Examples: Matrix multiplication using vector C intrinsics

In the Coral NPU GitHub repository, the directory /tests/cocotb/rvv/ml_ops/ contains several examples of programming matrix multiplication operations. These examples run on the RVV vector execution engine of Coral NPU, not the specialized matrix execution engine. Using the vector execution engine maximizes performance, as compared to traditional scalar processor cores.

These examples use the C programming intrinsics for RISC-V RVV vector operations and some use inline assembly code.

Example 1: rvv_matmul.cc

#include <riscv_vector.h>
#include <stdint.h>

// Assume rhs is column major
extern "C" void MatMul(size_t lhs_rows, size_t inner, size_t rhs_cols, const int8_t* lhs,
                       const int8_t* rhs, int32_t* result) {
  size_t vlmax = __riscv_vsetvl_e8m2(inner);

  for (size_t r = 0; r < lhs_rows; r++) {
    int32_t* result_row = result + (r * rhs_cols);

    for (size_t c = 0; c < rhs_cols; c++) {
      const int8_t* lhs_row = lhs + (r * inner);
      const int8_t* rhs_col = rhs + (c * inner);
      vint32m8_t vacc = __riscv_vmv_v_x_i32m8(0, vlmax);
      vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);
      size_t k = inner;
      while (k) {
        size_t vl = __riscv_vsetvl_e8m2(k);

        vint8m2_t vrhs = __riscv_vle8_v_i8m2(rhs_col, vl);
        vint16m4_t vrhs16 = __riscv_vwadd_vx_i16m4(vrhs, 0, vl);
        rhs_col += vl;

        vint8m2_t vlhs = __riscv_vle8_v_i8m2(lhs_row, vl);
        vint16m4_t vlhs16 = __riscv_vwadd_vx_i16m4(vlhs, 0, vl);
        lhs_row += vl;

        vacc = __riscv_vwmacc_vv_i32m8(vacc, vlhs16, vrhs16, vl);
        k -= vl;
      }

      vint32m1_t vres = __riscv_vredsum_vs_i32m8_i32m1(vacc, vzero, vlmax);
      __riscv_vse32_v_i32m1(result_row + c, vres, 1);
    }
  }
}

Example 2: rvv_matmul_assembly.cc

Using inline assembly code (with asm volatile) helps avoid compiler-generated spilling and helps register control.

#include <riscv_vector.h>
#include <stdint.h>

// Assume rhs is column major
extern "C" void MatMul(size_t lhs_rows, size_t inner, size_t rhs_cols, const int8_t* lhs,
                       const int8_t* rhs, int32_t* result) {
  size_t vlmax;
  asm volatile("vsetvli %0, %1, e8, m2, ta, ma" : "=r"(vlmax) : "r"(inner));

  for (size_t r = 0; r < lhs_rows; r++) {
    int32_t* result_row = result + (r * rhs_cols);

    for (size_t c = 0; c < rhs_cols; c++) {
      const int8_t* lhs_row = lhs + (r * inner);
      const int8_t* rhs_col = rhs + (c * inner);
      size_t k = inner;

      // Initialize v16 accumulator (e32m8) and v1 scalar reduction source to
      // zero
      asm volatile(
          "vsetvli zero, %0, e32, m8, ta, ma;\n\t"
          "vmv.v.i v16, 0;\n\t"
          "vsetivli zero, 1, e32, m1, ta, ma;\n\t"
          "vmv.v.i v1, 0"
          :
          : "r"(vlmax));

      while (k > 0) {
        size_t vl;
        asm volatile("vsetvli %0, %1, e8, m2, ta, ma" : "=r"(vl) : "r"(k));

        // Load inputs (v4, v6), widen to e16m4 (v8, v12), multiply-accumulate
        // into v16 (e32m8)
        asm volatile(
            "vsetvli zero, %0, e8, m2, ta, ma;\n\t"
            "vle8.v v4, (%1);\n\t"
            "vle8.v v6, (%2);\n\t"
            "vwadd.vx v8, v4, zero;\n\t"
            "vwadd.vx v12, v6, zero;\n\t"
            "vsetvli zero, %0, e16, m4, ta, ma;\n\t"
            "vwmacc.vv v16, v8, v12"
            :
            : "r"(vl), "r"(lhs_row), "r"(rhs_col));

        lhs_row += vl;
        rhs_col += vl;
        k -= vl;
      }

      // Reduce v16 over vlmax into v1, store scalar result
      asm volatile(
          "vsetvli zero, %0, e32, m8, ta, ma;\n\t"
          "vredsum.vs v1, v16, v1;\n\t"
          "vsetivli zero, 1, e32, m1, ta, ma;\n\t"
          "vse32.v v1, (%1)"
          :
          : "r"(vlmax), "r"(result_row + c));
    }
  }
}

More examples

These additional matmul examples may also be helpful:

Vector C intrinsics from RISC-V

This RISC-V document specifies the C language intrinsics for programming the RISC-V vector (RVV) extensions. This document is the definitive guide for developers who want to harness RISC-V vector capabilities directly from C code without dropping into assembly.

To leverage the C intrinsics in your toolchain, the header file riscv_vector.h must be included.

The supported RISC-V vector architecture, which is Zve32x for Coral NPU, is specified to the compiler. The standard vector extensions provides a set of smaller extensions for embedded use.

Control of the vector extension programming model

The RVV intrinsics allow users to control the fields in vtype, as well as the rounding modes for fixed-point (vxrm) and floating-point (frm) vector computations.

Control of effective element width (EEW) and effective LMUL (EMUL)

The RISC-V vector intrinsics data types are strongly-typed. The vector intrinsics encode the EEW (effective-element-width) and EMUL (effective LMUL) of the destination vector register in the suffix of the function name.