From 2efd1a6c14d2cd323374819a5cbe74e3547245b5 Mon Sep 17 00:00:00 2001 From: Samuel Akopyan Date: Tue, 4 Nov 2025 17:12:26 +0200 Subject: [PATCH 1/6] 388 - created new Parameter class, interface and test --- src/NeuralNet/Optimizers/Base/Optimizer.php | 29 ++++++ src/NeuralNet/Parameters/Parameter.php | 99 ++++++++++++++++++++ tests/NeuralNet/Parameters/ParameterTest.php | 49 ++++++++++ 3 files changed, 177 insertions(+) create mode 100644 src/NeuralNet/Optimizers/Base/Optimizer.php create mode 100644 src/NeuralNet/Parameters/Parameter.php create mode 100644 tests/NeuralNet/Parameters/ParameterTest.php diff --git a/src/NeuralNet/Optimizers/Base/Optimizer.php b/src/NeuralNet/Optimizers/Base/Optimizer.php new file mode 100644 index 000000000..0815cf0ce --- /dev/null +++ b/src/NeuralNet/Optimizers/Base/Optimizer.php @@ -0,0 +1,29 @@ + + */ +interface Optimizer extends Stringable +{ + /** + * Take a step of gradient descent for a given parameter. + * + * @internal + * + * @param Parameter $param + * @param NDArray $gradient + * @return NDArray + */ + public function step(Parameter $param, NDArray $gradient) : NDArray; +} diff --git a/src/NeuralNet/Parameters/Parameter.php b/src/NeuralNet/Parameters/Parameter.php new file mode 100644 index 000000000..efa7cf88a --- /dev/null +++ b/src/NeuralNet/Parameters/Parameter.php @@ -0,0 +1,99 @@ + + */ + +/** + * Parameter + * + */ +class Parameter +{ + /** + * The auto incrementing id. + * + * @var int + */ + protected static int $counter = 0; + + /** + * The unique identifier of the parameter. + * + * @var int + */ + protected int $id; + + /** + * The parameter. + * + * @var NDArray + */ + protected NDArray $param; + + /** + * @param NDArray $param + */ + public function __construct(NDArray $param) + { + $this->id = self::$counter++; + $this->param = $param; + } + + /** + * Return the unique identifier of the parameter. + * + * @return int + */ + public function id(): int + { + return $this->id; + } + + /** + * Return the wrapped parameter. + * + * @return NDArray + */ + public function param(): NDArray + { + return $this->param; + } + + /** + * Update the parameter with the gradient and optimizer. + * + * @param NDArray $gradient + * @param Optimizer $optimizer + */ + public function update(NDArray $gradient, Optimizer $optimizer): void + { + $step = $optimizer->step($this, $gradient); + + $this->param = NumPower::subtract($this->param, $step); + } + + /** + * Perform a deep copy of the object upon cloning. + */ + public function __clone(): void + { + $this->param = clone $this->param; + } +} diff --git a/tests/NeuralNet/Parameters/ParameterTest.php b/tests/NeuralNet/Parameters/ParameterTest.php new file mode 100644 index 000000000..b9373675c --- /dev/null +++ b/tests/NeuralNet/Parameters/ParameterTest.php @@ -0,0 +1,49 @@ +param = new Parameter(NumPower::array([ + [5, 4], + [-2, 6], + ])); + + $this->optimizer = new Stochastic(); + } + + public function testUpdate() : void + { + $gradient = NumPower::array([ + [2, 1], + [1, -2], + ]); + + $expected = [ + [4.98, 3.99], + [-2.01, 6.02], + ]; + + $this->param->update(gradient: $gradient, optimizer: $this->optimizer); + + self::assertEquals($expected, $this->param->param()->asArray()); + } +} From 3dc8300433410b8f498cd04bd93da32f0d8bbbbc Mon Sep 17 00:00:00 2001 From: Samuel Akopyan Date: Tue, 4 Nov 2025 17:21:04 +0200 Subject: [PATCH 2/6] 388 - Stochastic optimizer converted to NumPower --- .../Optimizers/Stochastic/Stochastic.php | 68 ++++++++++++++++++ .../Optimizers/Stochastic/StochasticTest.php | 70 +++++++++++++++++++ tests/NeuralNet/Parameters/ParameterTest.php | 6 +- 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 src/NeuralNet/Optimizers/Stochastic/Stochastic.php create mode 100644 tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php diff --git a/src/NeuralNet/Optimizers/Stochastic/Stochastic.php b/src/NeuralNet/Optimizers/Stochastic/Stochastic.php new file mode 100644 index 000000000..169cead69 --- /dev/null +++ b/src/NeuralNet/Optimizers/Stochastic/Stochastic.php @@ -0,0 +1,68 @@ + + */ +class Stochastic implements Optimizer +{ + /** + * The learning rate that controls the global step size. + * + * @var float + */ + protected float $rate; + + /** + * @param float $rate + * @throws InvalidArgumentException + */ + public function __construct(float $rate = 0.01) + { + if ($rate <= 0.0) { + throw new InvalidArgumentException("Learning rate must be greater than 0, $rate given."); + } + + $this->rate = $rate; + } + + /** + * Take a step of gradient descent for a given parameter. + * + * @internal + * + * @param Parameter $param + * @param NDArray $gradient + * @return NDArray + */ + public function step(Parameter $param, NDArray $gradient) : NDArray + { + return NumPower::multiply($gradient, $this->rate); + } + + /** + * Return the string representation of the object. + * + * @internal + * + * @return string + */ + public function __toString() : string + { + return "Stochastic (rate: {$this->rate})"; + } +} diff --git a/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php b/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php new file mode 100644 index 000000000..1dabfb8a6 --- /dev/null +++ b/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php @@ -0,0 +1,70 @@ +optimizer = new Stochastic(0.001); + } + + #[Test] + #[TestDox('Can be cast to a string')] + public function testToString() : void + { + self::assertEquals('Stochastic (rate: 0.001)', (string) $this->optimizer); + } + + /** + * @param Parameter $param + * @param NDArray $gradient + * @param list> $expected + */ + #[DataProvider('stepProvider')] + public function testStep(Parameter $param, NDArray $gradient, array $expected) : void + { + $step = $this->optimizer->step(param: $param, gradient: $gradient); + + self::assertEqualsWithDelta($expected, $step->toArray(), 1e-7); + } +} diff --git a/tests/NeuralNet/Parameters/ParameterTest.php b/tests/NeuralNet/Parameters/ParameterTest.php index b9373675c..9d0af9321 100644 --- a/tests/NeuralNet/Parameters/ParameterTest.php +++ b/tests/NeuralNet/Parameters/ParameterTest.php @@ -7,9 +7,9 @@ use NumPower; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\Group; -use Rubix\ML\NeuralNet\Optimizers\Optimizer; +use Rubix\ML\NeuralNet\Optimizers\Base\Optimizer; use Rubix\ML\NeuralNet\Parameters\Parameter; -use Rubix\ML\NeuralNet\Optimizers\Stochastic; +use Rubix\ML\NeuralNet\Optimizers\Stochastic\Stochastic; use PHPUnit\Framework\TestCase; #[Group('Parameters')] @@ -44,6 +44,6 @@ public function testUpdate() : void $this->param->update(gradient: $gradient, optimizer: $this->optimizer); - self::assertEquals($expected, $this->param->param()->asArray()); + self::assertEqualsWithDelta($expected, $this->param->param()->toArray(), 1e-7); } } From 0b7fe6241d24717090a185d9599a343cdae92506 Mon Sep 17 00:00:00 2001 From: Samuel Akopyan Date: Tue, 4 Nov 2025 18:47:08 +0200 Subject: [PATCH 3/6] 388 - added additional check in StochasticTest --- .../NeuralNet/Optimizers/Stochastic/StochasticTest.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php b/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php index 1dabfb8a6..06fe86434 100644 --- a/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php +++ b/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php @@ -13,6 +13,7 @@ use PHPUnit\Framework\Attributes\Test; use PHPUnit\Framework\Attributes\TestDox; use PHPUnit\Framework\TestCase; +use Rubix\ML\Exceptions\InvalidArgumentException; use Rubix\ML\NeuralNet\Parameters\Parameter; use Rubix\ML\NeuralNet\Optimizers\Stochastic\Stochastic; @@ -48,6 +49,15 @@ protected function setUp() : void $this->optimizer = new Stochastic(0.001); } + #[Test] + #[TestDox('Throws exception when constructed with invalid learning rate')] + public function testConstructorWithInvalidRate() : void + { + $this->expectException(InvalidArgumentException::class); + + new Stochastic(0.0); + } + #[Test] #[TestDox('Can be cast to a string')] public function testToString() : void From 477b0bef0413b6449a412d0fccc8561c6e4a4f24 Mon Sep 17 00:00:00 2001 From: Samuel Akopyan Date: Tue, 4 Nov 2025 18:47:44 +0200 Subject: [PATCH 4/6] 388 - updated docs for Stochastic class --- docs/neural-network/optimizers/stochastic.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/neural-network/optimizers/stochastic.md b/docs/neural-network/optimizers/stochastic.md index 3fc10afb5..f1fb53be0 100644 --- a/docs/neural-network/optimizers/stochastic.md +++ b/docs/neural-network/optimizers/stochastic.md @@ -1,4 +1,4 @@ -[source] +[source] # Stochastic A constant learning rate optimizer based on vanilla Stochastic Gradient Descent. @@ -10,7 +10,7 @@ A constant learning rate optimizer based on vanilla Stochastic Gradient Descent. ## Example ```php -use Rubix\ML\NeuralNet\Optimizers\Stochastic; +use Rubix\ML\NeuralNet\Optimizers\Stochastic\Stochastic; $optimizer = new Stochastic(0.01); -``` \ No newline at end of file +``` From 5f3863be94b5dbb5fbe5bf1c55bd338302f89994 Mon Sep 17 00:00:00 2001 From: Samuel Akopyan Date: Tue, 4 Nov 2025 18:51:27 +0200 Subject: [PATCH 5/6] 388 - updated docs for Stochastic test --- tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php b/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php index 06fe86434..57a50335f 100644 --- a/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php +++ b/tests/NeuralNet/Optimizers/Stochastic/StochasticTest.php @@ -37,9 +37,9 @@ public static function stepProvider() : Generator [0.04, -0.01, -0.5], ]), [ - [1e-5, 5e-5, -2e-5], - [-1e-5, 2e-5, 3e-5], - [4e-5, -1e-5, -0.0005], + [0.00001, 0.00005, -0.00002], + [-0.00001, 0.00002, 0.00003], + [0.00004, -0.00001, -0.0005], ], ]; } From c74c5fa93fb5d407858dae3b1a6af1187a9ff845 Mon Sep 17 00:00:00 2001 From: Samuel Akopyan Date: Tue, 4 Nov 2025 18:55:44 +0200 Subject: [PATCH 6/6] 388 - updated docs for Stochastic test --- docs/neural-network/optimizers/stochastic.md | 2 +- src/NeuralNet/Optimizers/Stochastic/Stochastic.php | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/neural-network/optimizers/stochastic.md b/docs/neural-network/optimizers/stochastic.md index f1fb53be0..4422e0ddc 100644 --- a/docs/neural-network/optimizers/stochastic.md +++ b/docs/neural-network/optimizers/stochastic.md @@ -1,7 +1,7 @@ [source] # Stochastic -A constant learning rate optimizer based on vanilla Stochastic Gradient Descent. +A constant learning rate optimizer based on vanilla Stochastic Gradient Descent (SGD). ## Parameters | # | Name | Default | Type | Description | diff --git a/src/NeuralNet/Optimizers/Stochastic/Stochastic.php b/src/NeuralNet/Optimizers/Stochastic/Stochastic.php index 169cead69..ffd9daf30 100644 --- a/src/NeuralNet/Optimizers/Stochastic/Stochastic.php +++ b/src/NeuralNet/Optimizers/Stochastic/Stochastic.php @@ -11,7 +11,8 @@ /** * Stochastic * - * A constant learning rate gradient descent optimizer. + * SGD (Stochastic Gradient Descent) optimizer - + * a constant learning rate gradient descent optimizer. * * @category Machine Learning * @package Rubix/ML