use cublaslt

[lucasavila00]

fn lt_mul(x: &Tensor, w: &QMatMul, lt: CublasLt) -> Result<Tensor> {
    // w.forward(x)
    match w {
        QMatMul::QTensor(ref qt) => {
            let w = qt.dequantize(x.device())?;

            let w = match *x.dims() {
                [b1, b2, _, _] => w.broadcast_left((b1, b2))?,
                [bsize, _, _] => w.broadcast_left(bsize)?,
                _ => w,
            };
            fused_batch_matmul(&w, &x, None, None, None, None, None, lt)

            // let w = match *x.dims() {
            //     [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?,
            //     [bsize, _, _] => w.broadcast_left(bsize)?.t()?,
            //     _ => w.t()?,
            // };
            // x.matmul(&w)
        }
        QMatMul::Tensor(_) => todo!(),
    }
}

fn lt_mul(x: &Tensor, w: &QMatMul, lt: CublasLt) -> Result<Tensor> {
    // w.forward(x)
    match w {
        QMatMul::QTensor(ref qt) => {
            let w = qt.dequantize(x.device())?;

            // let w = match *x.dims() {
            //     [b1, b2, _, _] => w.broadcast_left((b1, b2))?,
            //     [bsize, _, _] => w.broadcast_left(bsize)?,
            //     _ => w,
            // };
            // fused_batch_matmul(&w, &x, None, None, None, None, None, lt)

            let w = match *x.dims() {
                [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?,
                [bsize, _, _] => w.broadcast_left(bsize)?.t()?,
                _ => w.t()?,
            };
            x.matmul(&w)
        }
        QMatMul::Tensor(_) => todo!(),
    }
}

Candle already uses CublasLT

[github-actions]

Code Metrics Report

  ───────────────────────────────────────────────────────────────────────────────
Language                 Files     Lines   Blanks  Comments     Code Complexity
───────────────────────────────────────────────────────────────────────────────
Rust                        70     22354     1540       502    20312       1261
───────────────────────────────────────────────────────────────────────────────
Total                       70     22354     1540       502    20312       1261
───────────────────────────────────────────────────────────────────────────────
Estimated Cost to Develop 37,875
Estimated Schedule Effort 11.593474 months
Estimated People Required 4.888076
───────────────────────────────────────────────────────────────────────────────
Processed 738139 bytes, 0.738 megabytes (SI)
───────────────────────────────────────────────────────────────────────────────