Calculate dot product 2 times in different directions

[MarkKremer]

Hey people. 👋

I'm trying to calculate the dot product of a matrix + vector. I have to do this 2 times in different directions (so the matrix is used as input for 2 different operations in the graph). Everything is batched and differentiation is needed. I've tried different solutions using BatchedMatMul and Broadcast->HadamardProd->Sum but I'm running into difficulties with each of them.

Here's an example using BatchedMatMul:

package main

import (
	"fmt"
	"gorgonia.org/gorgonia"
	"gorgonia.org/tensor"
	"log"
)

func main() {
	g := gorgonia.NewGraph()

	m := gorgonia.NewTensor(g, tensor.Float64, 3, gorgonia.WithShape(1, 3, 4), gorgonia.WithName("m"), gorgonia.WithValue(tensor.New(tensor.WithShape(1, 3, 4), tensor.WithBacking([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}))))
	fmt.Printf("Matrix =\n%v", m.Value())
	// Matrix =
	//   ⎡ 1   2   3   4⎤
	//   ⎢ 5   6   7   8⎥
	//   ⎣ 9  10  11  12⎦

	a := gorgonia.NewTensor(g, tensor.Float64, 3, gorgonia.WithShape(1, 4, 1), gorgonia.WithName("a"), gorgonia.WithValue(tensor.New(tensor.WithShape(1, 4, 1), tensor.WithBacking([]float64{1, 2, 3, 4}))))
	fmt.Printf("a =\n%v", a.Value())
	// a =
	//   ⎡1⎤
	//   ⎢2⎥
	//   ⎣3⎥
	//   ⎢4⎦

	b := gorgonia.NewTensor(g, tensor.Float64, 3, gorgonia.WithShape(1, 1, 3), gorgonia.WithName("b"), gorgonia.WithValue(tensor.New(tensor.WithShape(1, 1, 3), tensor.WithBacking([]float64{1, 2, 3}))))
	fmt.Printf("b =\n%v", b.Value())
	// b =
	//   ⎡1  2  3⎤

	dot1, err := gorgonia.BatchedMatMul(m, a)
	if err != nil {
		log.Fatal(err)
	}

	dot2, err := gorgonia.BatchedMatMul(b, m)
	if err != nil {
		log.Fatal(err)
	}

	machine := gorgonia.NewTapeMachine(g)
	defer machine.Close()
	if err = machine.RunAll(); err != nil {
		log.Fatal(err)
	}

	fmt.Printf("dot1 =\n%v\n", dot1.Value())
	fmt.Printf("dot2 =\n%v\n", dot2.Value())
	
	//panic: blas: insufficient length of c
	//
	//goroutine 7 [running]:
	//gonum.org/v1/gonum/blas/gonum.Implementation.Dgemm(0x4e4e, 0x1, 0x4, 0x3, 0x3ff0000000000000, 0xc000017040, 0x3, 0x3, 0x3, 0xc00001a360, ...)
	//        /home/markkremer/go/pkg/mod/gonum.org/v1/gonum@v0.7.0/blas/gonum/dgemm.go:92 +0x57d
	//gorgonia.org/tensor.StdEng.MatMul(0xc05ce0, 0xc0001a25a0, 0xc05ce0, 0xc0001a26c0, 0xc05ce0, 0xc0001a27e0, 0x1, 0x7f33f628a068)
	//        /home/markkremer/go/pkg/mod/gorgonia.org/tensor@v0.9.6/defaultengine_linalg.go:529 +0x648
	//gorgonia.org/tensor.(*Dense).MatMul(0xc0001a25a0, 0xc05ce0, 0xc0001a26c0, 0xc000081b20, 0x1, 0x1, 0xc0001a27e0, 0x0, 0x0)
	//        /home/markkremer/go/pkg/mod/gorgonia.org/tensor@v0.9.6/dense_linalg.go:148 +0x2c5
	//gorgonia.org/tensor.MatMul(0xc05ce0, 0xc0001a25a0, 0xc05ce0, 0xc0001a26c0, 0xc000081b20, 0x1, 0x1, 0x0, 0x0, 0x0, ...)
	//        /home/markkremer/go/pkg/mod/gorgonia.org/tensor@v0.9.6/api_arith.go:577 +0x13a
	//gorgonia.org/gorgonia.batchedMatMul(0xc05ce0, 0xc0001525a0, 0xc05ce0, 0xc000152240, 0xc05ce0, 0xc0001a2480, 0x0, 0xc05ce0, 0x203000, 0x203000, ...)
	//        /home/markkremer/go/pkg/mod/gorgonia.org/gorgonia@v0.9.11-0.20200423095535-5fb5944d4a1f/operatorLinAlg.go:468 +0x556
	//gorgonia.org/gorgonia.linAlgBinOp.preallocBatchMatMul(0x4, 0xc02700, 0xc0001a2480, 0xc00015c6c0, 0x2, 0x2, 0x0, 0xb50bb0, 0xc00004a380, 0x7f33f853b6d0)
	//        /home/markkremer/go/pkg/mod/gorgonia.org/gorgonia@v0.9.11-0.20200423095535-5fb5944d4a1f/op_math.go:772 +0x1d7
	//gorgonia.org/gorgonia.linAlgBinOp.UsePreallocDo(0xc000000004, 0xc02700, 0xc0001a2480, 0xc00015c6c0, 0x2, 0x2, 0xac4ba0, 0xb1e1c0, 0xc000017301, 0x7f33f628a048)
	//        /home/markkremer/go/pkg/mod/gorgonia.org/gorgonia@v0.9.11-0.20200423095535-5fb5944d4a1f/op_math.go:712 +0xe5
	//gorgonia.org/gorgonia.(*execOp).exec(0xc00009a550, 0xc000182000, 0x0, 0x0)
	//        /home/markkremer/go/pkg/mod/gorgonia.org/gorgonia@v0.9.11-0.20200423095535-5fb5944d4a1f/vm_tape_nocuda.go:41 +0xddc
	//gorgonia.org/gorgonia.(*tapeMachine).runall(0xc000182000, 0xc00001a420, 0xc00001a480)
	//        /home/markkremer/go/pkg/mod/gorgonia.org/gorgonia@v0.9.11-0.20200423095535-5fb5944d4a1f/vm_tape.go:236 +0x10e
	//created by gorgonia.org/gorgonia.(*tapeMachine).RunAll
	//        /home/markkremer/go/pkg/mod/gorgonia.org/gorgonia@v0.9.11-0.20200423095535-5fb5944d4a1f/vm_tape.go:208 +0x163
	//
	//Process finished with exit code 2
}

If I do the BatchedMatMuls separately it works. I know Gorgonia does a lot of in-place magic but I don't know how to get around it.
In my own project this somehow works but then the differentiation doesn't like adding the gradients of 2 different shapes.

I'll add code to calculate the gradients when I can get past this error.

[MarkKremer]

Somehow it sets the shape of b to (1, 1, 1). It should be (1, 1, 3).

When I remove dot1 (which operates on m and a, not b), it sets b to (1, 1, 4), which is still not equal to (1, 1, 3).

Removing both dots:

Note that the order of the nodes changed in the images, which can be a bit confusing.

Increasing the batch size to 2 has the same result but with the first dim set to 2 everywhere.

[MarkKremer]

I think I found the cause of the bug. I will update this issue with a fix PR and/or further problems I encounter.

[wzzhu]

This works in latest build after this issue gorgonia/tensor#73 is fixed

[chewxy]

OK closing. Thanks all