From 6d12004f693adce14ff99208b65f81a875d9d22f Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 14 Jan 2024 22:30:03 +0200 Subject: [PATCH 1/3] Played with adaptive learning rate and gradient normalization --- main.go | 32 ++++++++++++++++++++++++-------- plot.go | 2 +- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index e39985b..285a9a4 100644 --- a/main.go +++ b/main.go @@ -3,26 +3,32 @@ package main import ( "image" "log" + "math" "time" "github.com/hajimehoshi/ebiten/v2" ) -func f(x float64) float64 { return x*x + 5*x - 3 } -func df(x float64) float64 { return 2*x + 5 } +const ( + screenWidth, screenHeight = 720, 480 + lr = 0.15 // Learning rate +) + +func f(x float64) float64 { return math.Cos(x) } +func df(x float64) float64 { return -math.Sin(x) } func main() { - ebiten.SetWindowSize(640, 480) + ebiten.SetWindowSize(screenWidth, screenHeight) ebiten.SetWindowTitle("Gradient descent") img := make(chan *image.RGBA, 1) go func() { - p := Plot(-5, 0, 0.1, f) - x := 0.0 + p := Plot(-10, 5, 0.1, f) + x := .25 img <- p(x) - for i := 0; i < 50; i++ { - time.Sleep(30 * time.Millisecond) - x -= df(x) * 0.1 + for i := 0; df(x) != 0; i++ { + time.Sleep(1000 * time.Millisecond) + x -= sign(df(x)) * 1 / math.Min(float64(i)+1, 100) img <- p(x) } }() @@ -31,3 +37,13 @@ func main() { log.Fatal(err) } } + +func sign(a float64) float64 { + switch { + case a < 0: + return -1 + case a > 0: + return +1 + } + return 0 +} diff --git a/plot.go b/plot.go index ebad8a9..d571d3f 100644 --- a/plot.go +++ b/plot.go @@ -52,7 +52,7 @@ func Plot(xmin, xmax, xstep float64, f func(float64) float64) func(x float64) *i p.X.Label.Text = "X" p.Y.Label.Text = "Y" - img := image.NewRGBA(image.Rect(0, 0, 640, 480)) + img := image.NewRGBA(image.Rect(0, 0, screenWidth, screenHeight)) c := vgimg.NewWith(vgimg.UseImage(img)) p.Draw(draw.New(c)) return c.Image().(*image.RGBA) From e88988854650d36dfe4541af83190c3442380c38 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 19 Jan 2024 15:16:35 +0200 Subject: [PATCH 2/3] Tried implementing approximation using gradient descent unsuccessfuly for now --- go.mod | 4 +- main.go | 149 +++++++++++++++++++++++++++++++++++++++++++++----------- plot.go | 105 ++++++++++++++++++++++----------------- 3 files changed, 182 insertions(+), 76 deletions(-) diff --git a/go.mod b/go.mod index fd11da7..88b34d3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ -module github.com/prog-1/gradient-descent +module gradient-descent -go 1.21.1 +go 1.21.6 require ( github.com/hajimehoshi/ebiten/v2 v2.6.3 diff --git a/main.go b/main.go index 285a9a4..ae98d1e 100644 --- a/main.go +++ b/main.go @@ -1,49 +1,142 @@ package main import ( + "fmt" "image" "log" - "math" - "time" + "math/rand" "github.com/hajimehoshi/ebiten/v2" + "gonum.org/v1/plot/plotter" ) const ( - screenWidth, screenHeight = 720, 480 - lr = 0.15 // Learning rate + screenWidth, screenHeight = 720, 480 + randMin, randMax = -10, 10 + epochs, lr = 5, 0.25 + plotMinX, plotMaxX, plotMinY, plotMaxY = -10, 10, -50, 100 // Min and Max data values along both axis + pointMinYOffset, pointMaxYOffset, pointCount = -20, 20, 10 ) -func f(x float64) float64 { return math.Cos(x) } -func df(x float64) float64 { return -math.Sin(x) } +// Function points are spawed along +func f(x float64) float64 { + return 0.5*x + 2 +} + +// Approximating function(af) = Inference for 1 argument(x) +func af(x, a, b float64) float64 { return a*x + b } + +// Runs model on all the input data +func inference(x []float64, w, b float64) (out []float64) { + for _, v := range x { + out = append(out, af(v, w, b)) + } + return +} + +func loss(labels, y []float64) float64 { + var errSum float64 + for i := range labels { + errSum += (y[i] - labels[i]) * (y[i] - labels[i]) + } + return errSum / float64(len(labels)) // For the sake of making numbers smaller -> better percievable +} + +func gradient(labels, y, x []float64) (dw, db float64) { + // dw, db - Parial derivatives, w - weight, b - bias + for i := 0; i < len(labels); i++ { + dif := y[i] - labels[i] + dw += dif * x[i] + db += dif + } + n := float64(len(labels)) + dw *= 2 / n + db *= 2 / n + + return +} + +func train(epochs int, inputs, labels []float64) (w, b float64) { + // randFloat64 := func() float64 { + // return randMin + rand.Float64()*(randMax-randMin) + // } + // w, b = randFloat64(), randFloat64() + w, b = 1, 0 + var dw, db float64 + for i := 0; i < epochs; i++ { + dw, db = gradient(labels, inference(inputs, w, b), inputs) + w -= dw * lr + b -= db * lr + } + return +} + +// Returns random points along f with random Yoffset +func randPoints(f func(float64) float64, minYoffset float64, maxYoffset float64, pointCount uint) (xs, labels []float64) { + // 1. Getting random argument value X + // 2. Getting function value(Y) + // 3. Applying offset to Y + for i := uint(0); i < pointCount; i++ { + x := plotMinX + rand.Float64()*(plotMaxX-plotMinX) // Random argument within visible range + yOffset := minYoffset + rand.Float64()*(maxYoffset-minYoffset) + xs = append(xs, x) + labels = append(labels, f(x)+yOffset) + } + return +} + +// func main() { +// xs, labels := randPoints(f, pointMinYOffset, pointMaxYOffset, pointCount) +// w, b := train(epochs, labels, xs) +// +// img := make(chan *image.RGBA, 1) +// go func() { +// p := Plot(plotMinX, plotMaxX, ) +// } +// if err := ebiten.RunGame(&App{Img: img}); err != nil { +// log.Fatal(err) +// } +// } + +// func main() { +// var loss plotter.XYs +// +// for i := 0; i < epochs; i++ { +// y := inference(inputs, w, b) +// loss = append(loss, plotter.XY{ +// X: float64(i), +// Y: msl(labels, y), +// }) +// lossLines, _ := plotter.NewLine(loss) +// if plotLoss { +// select { +// case img <- Plot(lossLines): +// default: +// } +// } else { +// const extra = (inputPointsMaxX - inputPointsMinX) / 10 +// xs := []float64{inputPointsMinX - extra, inputPointsMaxX + extra} +// ys := inference(xs, w, b) +// resLine, _ := plotter.NewLine(plotter.XYs{{X: xs[0], Y: ys[0]}, {X: xs[1], Y: ys[1]}}) +// img <- Plot(inputsScatter, resLine) +// } func main() { - ebiten.SetWindowSize(screenWidth, screenHeight) - ebiten.SetWindowTitle("Gradient descent") + inputs, labels := randPoints(f, pointMinYOffset, pointMaxYOffset, pointCount) + var points plotter.XYs + for i := 0; i < pointCount; i++ { + points = append(points, plotter.XY{X: inputs[i], Y: labels[i]}) + } img := make(chan *image.RGBA, 1) - go func() { - p := Plot(-10, 5, 0.1, f) - x := .25 - img <- p(x) - for i := 0; df(x) != 0; i++ { - time.Sleep(1000 * time.Millisecond) - x -= sign(df(x)) * 1 / math.Min(float64(i)+1, 100) - img <- p(x) - } - }() + pointsScatter, _ := plotter.NewScatter(points) + fp := plotter.NewFunction(f) // f plot + w, b := train(epochs, inputs, labels) + fmt.Println(w, b) + ap := plotter.NewFunction(func(x float64) float64 { return w*x + b }) // approximating function plot + img <- Plot(pointsScatter, fp, ap) if err := ebiten.RunGame(&App{Img: img}); err != nil { log.Fatal(err) } } - -func sign(a float64) float64 { - switch { - case a < 0: - return -1 - case a > 0: - return +1 - } - return 0 -} diff --git a/plot.go b/plot.go index d571d3f..0d363b5 100644 --- a/plot.go +++ b/plot.go @@ -1,60 +1,73 @@ package main import ( - "fmt" "image" - "image/color" - "log" "gonum.org/v1/plot" "gonum.org/v1/plot/plotter" - "gonum.org/v1/plot/vg" "gonum.org/v1/plot/vg/draw" "gonum.org/v1/plot/vg/vgimg" ) -func Plot(xmin, xmax, xstep float64, f func(float64) float64) func(x float64) *image.RGBA { - var pts plotter.XYs - for x := xmin; x <= xmax; x += xstep { - pts = append(pts, plotter.XY{X: x, Y: f(x)}) - } - fn, err := plotter.NewLine(pts) - if err != nil { - log.Fatalf("Failed to NewLine: %v", err) - } - fn.Color = color.RGBA{B: 255, A: 255} - return func(x float64) *image.RGBA { - pts := plotter.XYs{{X: x, Y: f(x)}} - xScatter, err := plotter.NewScatter(pts) - if err != nil { - log.Fatalf("Failed to NewScatter: %v", err) - } - xScatter.Color = color.RGBA{R: 255, A: 255} +//// Old one +// func Plot(xmin, xmax, xstep float64, f func(float64) float64) func(x float64) *image.RGBA { +// var pts plotter.XYs +// for x := xmin; x <= xmax; x += xstep { +// pts = append(pts, plotter.XY{X: x, Y: f(x)}) +// } +// fn, err := plotter.NewLine(pts) +// if err != nil { +// log.Fatalf("Failed to NewLine: %v", err) +// } +// fn.Color = color.RGBA{B: 255, A: 255} +// return func(x float64) *image.RGBA { +// pts := plotter.XYs{{X: x, Y: f(x)}} +// xScatter, err := plotter.NewScatter(pts) +// if err != nil { +// log.Fatalf("Failed to NewScatter: %v", err) +// } +// xScatter.Color = color.RGBA{R: 255, A: 255} +// +// labels, err := plotter.NewLabels(plotter.XYLabels{ +// XYs: pts, +// Labels: []string{fmt.Sprintf("x = %.2f", x)}, +// }) +// labels.Offset = vg.Point{X: -10, Y: 15} +// if err != nil { +// log.Fatalf("Failed to NewLabels: %v", err) +// } +// +// p := plot.New() +// p.Add( +// plotter.NewGrid(), +// fn, +// xScatter, +// labels, +// ) +// p.Legend.Add("f(x)", fn) +// p.Legend.Add("x", xScatter) +// p.X.Label.Text = "X" +// p.Y.Label.Text = "Y" +// +// img := image.NewRGBA(image.Rect(0, 0, screenWidth, screenHeight)) +// c := vgimg.NewWith(vgimg.UseImage(img)) +// p.Draw(draw.New(c)) +// return c.Image().(*image.RGBA) +// } +// } - labels, err := plotter.NewLabels(plotter.XYLabels{ - XYs: pts, - Labels: []string{fmt.Sprintf("x = %.2f", x)}, - }) - labels.Offset = vg.Point{X: -10, Y: 15} - if err != nil { - log.Fatalf("Failed to NewLabels: %v", err) - } +func Plot(ps ...plot.Plotter) *image.RGBA { + p := plot.New() + p.X.Min = plotMinX + p.X.Max = plotMaxX + p.Y.Min = plotMinY + p.Y.Max = plotMaxY - p := plot.New() - p.Add( - plotter.NewGrid(), - fn, - xScatter, - labels, - ) - p.Legend.Add("f(x)", fn) - p.Legend.Add("x", xScatter) - p.X.Label.Text = "X" - p.Y.Label.Text = "Y" - - img := image.NewRGBA(image.Rect(0, 0, screenWidth, screenHeight)) - c := vgimg.NewWith(vgimg.UseImage(img)) - p.Draw(draw.New(c)) - return c.Image().(*image.RGBA) - } + p.Add(append([]plot.Plotter{ + plotter.NewGrid(), + }, ps...)...) + img := image.NewRGBA(image.Rect(0, 0, screenWidth, screenHeight)) + c := vgimg.NewWith(vgimg.UseImage(img)) + p.Draw(draw.New(c)) + return c.Image().(*image.RGBA) } From ef6e3689c7c673ff08c7d757b89acfee31ae9d09 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 20 Jan 2024 14:15:08 +0200 Subject: [PATCH 3/3] Fixed program by tweaking learning rate --- main.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index ae98d1e..32b33a4 100644 --- a/main.go +++ b/main.go @@ -13,23 +13,24 @@ import ( const ( screenWidth, screenHeight = 720, 480 randMin, randMax = -10, 10 - epochs, lr = 5, 0.25 + epochs, lr = 1000000, 0.000005 plotMinX, plotMaxX, plotMinY, plotMaxY = -10, 10, -50, 100 // Min and Max data values along both axis pointMinYOffset, pointMaxYOffset, pointCount = -20, 20, 10 ) // Function points are spawed along func f(x float64) float64 { - return 0.5*x + 2 + // return 0.5*x + 2 + return 10*x - 5 } -// Approximating function(af) = Inference for 1 argument(x) -func af(x, a, b float64) float64 { return a*x + b } +// Inference for 1 argument(x) +func i(x, w, b float64) float64 { return w*x + b } // Runs model on all the input data func inference(x []float64, w, b float64) (out []float64) { for _, v := range x { - out = append(out, af(v, w, b)) + out = append(out, i(v, w, b)) } return } @@ -57,16 +58,17 @@ func gradient(labels, y, x []float64) (dw, db float64) { } func train(epochs int, inputs, labels []float64) (w, b float64) { - // randFloat64 := func() float64 { - // return randMin + rand.Float64()*(randMax-randMin) - // } - // w, b = randFloat64(), randFloat64() - w, b = 1, 0 + randFloat64 := func() float64 { + return randMin + rand.Float64()*(randMax-randMin) + } + w, b = randFloat64(), randFloat64() + // w, b = 1, 0 var dw, db float64 for i := 0; i < epochs; i++ { dw, db = gradient(labels, inference(inputs, w, b), inputs) w -= dw * lr b -= db * lr + fmt.Println(w, b) } return }