more refactoring

src/main.rs

@@ -2,6 +2,8 @@ pub mod dataset;
 pub mod neural_network;
 pub mod plotter;
 
+use neural_network::layer::{DenseLayer, Layer};
+
 #[allow(unused_imports)]
 use crate::{
     dataset::example_datasets::{CIRCLE, RGB_DONUT, RGB_TEST, XOR},
@@ -17,19 +19,19 @@ use crate::{
 #[allow(dead_code)]
 fn main() {
     //Define Network Shape
-    let network_shape: Vec<(&dyn ActivationFunction, usize)> = vec![
-        (&Relu, 2),
-        (&Relu, 32),
-        (&Relu, 32),
-        (&Relu, 32),
-        (&Relu, 3),
+    let layers: Vec<Box<dyn Layer>> = vec![
+        Box::new(DenseLayer::new(2, &Relu)),
+        Box::new(DenseLayer::new(32, &Relu)),
+        Box::new(DenseLayer::new(32, &Sigmoid)),
+        Box::new(DenseLayer::new(32, &Relu)),
+        Box::new(DenseLayer::new(3, &Linear)),
     ];
 
     //Define Optimizer
     let mut optimizer = ADAM::default();
 
     //Create Network
-    let mut network = Network::new(&network_shape, &mut optimizer, &QuadraticCost);
+    let mut network = Network::new(layers, &mut optimizer, &QuadraticCost);
 
     //Define Dataset
     let dataset = &RGB_DONUT;

src/neural_network.rs

@@ -7,38 +7,42 @@ use crate::dataset::Dataset;
 
 use ndarray::Array2;
 
-use self::{
-    activation_function::ActivationFunction, cost_function::CostFunction, layer::Layer,
-    optimizer::Optimizer,
-};
+use self::{cost_function::CostFunction, layer::Layer, optimizer::Optimizer};
 
 pub struct Network<'a> {
-    pub layers: Vec<Layer>,
+    input_size: usize,
+    output_size: usize,
+    layers: Vec<Box<dyn Layer>>,
     optimizer: &'a mut dyn Optimizer,
-    pub shape: &'a [(&'static dyn ActivationFunction, usize)],
-    pub cost_function: &'static dyn CostFunction,
+    cost_function: &'static dyn CostFunction,
 }
 
 #[allow(non_snake_case)]
 impl Network<'_> {
     pub fn new<'a>(
-        shape: &'a [(&'static dyn ActivationFunction, usize)],
+        mut layers: Vec<Box<dyn Layer>>,
         optimizer: &'a mut dyn Optimizer,
         cost_function: &'static dyn CostFunction,
     ) -> Network<'a> {
-        let mut layers = Vec::new();
-        for i in 0..shape.len() - 1 {
-            let (activation_function, input_size) = shape[i];
-            let (_, output_size) = shape[i + 1];
-            layers.push(Layer::new(input_size, output_size, activation_function));
+        // Initialize the layers
+        let network_shape = layers.iter().map(|l| l.get_size()).collect::<Vec<_>>();
+
+        //remove last layer
+        layers.pop();
+
+        //initialize the layers
+        for (i, boxedLeyer) in layers.iter_mut().enumerate() {
+            let layer = boxedLeyer.as_mut();
+            layer.initialize(network_shape[i], network_shape[i + 1]);
         }
 
         optimizer.initialize(&layers);
 
         Network {
+            input_size: network_shape[0],
+            output_size: network_shape[network_shape.len() - 1],
             layers,
             optimizer,
-            shape,
             cost_function,
         }
     }
@@ -68,7 +72,7 @@ impl Network<'_> {
         for layer in &self.layers {
             let z = layer.forward(&activation);
             zs.push(z.clone());
-            activation = layer.activation.f_array(&z);
+            activation = layer.get_activation().f_array(&z);
             activations.push(activation.clone());
         }
 
@@ -77,7 +81,7 @@ impl Network<'_> {
 
         // Calculate sensitivity
         let sig_prime = self.layers[self.layers.len() - 1]
-            .activation
+            .get_activation()
             .d_array(&zs[zs.len() - 1]);
 
         // Calculate delta for last layer
@@ -88,12 +92,12 @@ impl Network<'_> {
         nabla_ws.push((&activations[activations.len() - 2]).t().dot(&delta));
 
         // Loop backwards through the layers, calculating delta, nabla_b and nabla_w
-        for i in 2..self.shape.len() {
+        for i in 2..self.layers.len() + 1 {
             let sig_prime = self.layers[self.layers.len() - i]
-                .activation
+                .get_activation()
                 .d_array(&zs[zs.len() - i]);
 
-            let nabla_c = &delta.dot(&self.layers[self.layers.len() - i + 1].weights.t());
+            let nabla_c = &delta.dot(&self.layers[self.layers.len() - i + 1].get_weights().t());
 
             delta = nabla_c * sig_prime;
 
@@ -125,12 +129,12 @@ impl Network<'_> {
         //assert iput shape is the same as the data
         assert_eq!(
             X.ncols(),
-            self.shape[0].1,
+            self.input_size,
             "Input shape does not match data"
         );
         assert_eq!(
             y.ncols(),
-            self.shape[self.shape.len() - 1].1,
+            self.output_size,
             "Output shape does not match data"
         );
 
@@ -199,7 +203,7 @@ pub trait Summary {
 
 impl Summary for Network<'_> {
     fn summerize(&self) -> String {
-        let shape = self.shape.iter().map(|x| x.1).collect::<Vec<_>>();
+        let shape = self.layers.iter().map(|x| x.get_size()).collect::<Vec<_>>();
 
         format!("{}_{:?}", self.optimizer.summerize(), shape).replace(" ", "")
     }

src/neural_network/layer.rs

@@ -3,37 +3,79 @@ use ndarray_rand::{rand_distr::Normal, RandomExt};
 
 use super::activation_function::ActivationFunction;
 
-pub struct Layer {
+pub trait Layer {
+    fn new(input_size: usize, activation: &'static dyn ActivationFunction) -> Self
+    where
+        Self: Sized;
+
+    fn initialize(&mut self, input_size: usize, output_size: usize);
+
+    fn predict(&self, input: &Array2<f64>) -> Array2<f64>;
+    fn forward(&self, input: &Array2<f64>) -> Array2<f64>;
+
+    fn get_size(&self) -> usize;
+    fn get_activation(&self) -> &'static dyn ActivationFunction;
+    fn get_weights(&self) -> &Array2<f64>;
+    fn set_weights(&mut self, weights: Array2<f64>);
+    fn get_bias(&self) -> &Array2<f64>;
+    fn set_bias(&mut self, biases: Array2<f64>);
+}
+
+pub struct DenseLayer {
+    pub input_size: usize,
     pub weights: Array2<f64>,
     pub biases: Array2<f64>,
     pub activation: &'static dyn ActivationFunction,
 }
 
-impl Layer {
-    pub fn new(
-        input_size: usize,
-        output_size: usize,
-        activation: &'static dyn ActivationFunction,
-    ) -> Layer {
-        let weights = Array2::random((input_size, output_size), Normal::new(0.0, 1.0).unwrap())
-            / (input_size as f64).sqrt();
-        let biases = Array2::random((1, output_size), Normal::new(0.0, 0.1).unwrap());
-
-        Layer {
-            weights,
-            biases,
+impl Layer for DenseLayer {
+    fn new(input_size: usize, activation: &'static dyn ActivationFunction) -> DenseLayer {
+        DenseLayer {
+            input_size,
             activation,
+            weights: Array2::zeros((0, 0)),
+            biases: Array2::zeros((0, 0)),
         }
     }
 
+    fn initialize(&mut self, input_size: usize, output_size: usize) {
+        self.weights = Array2::random((input_size, output_size), Normal::new(0.0, 1.0).unwrap())
+            / (input_size as f64).sqrt();
+        self.biases = Array2::random((1, output_size), Normal::new(0.0, 0.1).unwrap());
+    }
+
+    fn get_size(&self) -> usize {
+        self.input_size
+    }
+
+    fn get_activation(&self) -> &'static dyn ActivationFunction {
+        self.activation
+    }
+
+    fn get_weights(&self) -> &Array2<f64> {
+        &self.weights
+    }
+
+    fn get_bias(&self) -> &Array2<f64> {
+        &self.biases
+    }
+
+    fn set_weights(&mut self, weights: Array2<f64>) {
+        self.weights = weights;
+    }
+
+    fn set_bias(&mut self, biases: Array2<f64>) {
+        self.biases = biases;
+    }
+
     // Predicts the output of the layer given an input
-    pub fn predict(&self, input: &Array2<f64>) -> Array2<f64> {
+    fn predict(&self, input: &Array2<f64>) -> Array2<f64> {
         let a = &self.forward(input);
         self.activation.f_array(a)
     }
 
     // Calculates the weighted sum of the input
-    pub fn forward(&self, input: &Array2<f64>) -> Array2<f64> {
+    fn forward(&self, input: &Array2<f64>) -> Array2<f64> {
         input.dot(&self.weights) + &self.biases
     }
 }

src/neural_network/optimizer.rs

@@ -9,12 +9,12 @@ use crate::neural_network::{layer::Layer, Summary};
 pub trait Optimizer: Summary {
     fn update_params(
         &mut self,
-        layers: &mut Vec<Layer>,
+        layers: &mut Vec<Box<dyn Layer>>,
         nabla_bs: &Vec<Array2<f64>>,
         nabla_ws: &Vec<Array2<f64>>,
     );
 
-    fn initialize(&mut self, layers: &Vec<Layer>);
+    fn initialize(&mut self, layers: &Vec<Box<dyn Layer>>);
 
     fn pre_update(&mut self);
 

src/neural_network/optimizer/adam_optimizer.rs

@@ -45,8 +45,7 @@ impl ADAM {
 impl Optimizer for ADAM {
     fn update_params(
         &mut self,
-        layers: &mut Vec<Layer>,
-
+        layers: &mut Vec<Box<dyn Layer>>,
         nabla_bs: &Vec<Array2<f64>>,
         nabla_ws: &Vec<Array2<f64>>,
     ) {
@@ -81,18 +80,22 @@ impl Optimizer for ADAM {
                 / (biases_cache_corrected.mapv(f64::sqrt) + self.epsilon);
 
             //updates
-            layer.weights = &layer.weights - &weights_update;
-            layer.biases = &layer.biases - &biases_update;
+
+            layer.set_weights(layer.get_weights() - &weights_update);
+            layer.set_bias(layer.get_bias() - &biases_update);
         }
     }
 
-    fn initialize(&mut self, layers: &Vec<Layer>) {
+    fn initialize(&mut self, layers: &Vec<Box<dyn Layer>>) {
         for layer in layers {
-            self.weights_cache.push(Array2::zeros(layer.weights.dim()));
-            self.biases_cache.push(Array2::zeros(layer.biases.dim()));
+            self.weights_cache
+                .push(Array2::zeros(layer.get_weights().dim()));
+            self.biases_cache
+                .push(Array2::zeros(layer.get_bias().dim()));
             self.weights_momentum
-                .push(Array2::zeros(layer.weights.dim()));
-            self.biases_momentum.push(Array2::zeros(layer.biases.dim()));
+                .push(Array2::zeros(layer.get_weights().dim()));
+            self.biases_momentum
+                .push(Array2::zeros(layer.get_bias().dim()));
         }
     }
 

src/neural_network/optimizer/rmsprop_optimizer.rs

@@ -39,8 +39,7 @@ impl RMS_PROP {
 impl Optimizer for RMS_PROP {
     fn update_params(
         &mut self,
-        layers: &mut Vec<Layer>,
-
+        layers: &mut Vec<Box<dyn Layer>>,
         nabla_bs: &Vec<Array2<f64>>,
         nabla_ws: &Vec<Array2<f64>>,
     ) {
@@ -58,15 +57,18 @@ impl Optimizer for RMS_PROP {
                 / (self.biases_cache[i].mapv(|x| x.sqrt()) + self.epsilon);
 
             //Update weights and biases
-            layer.weights = &layer.weights + weights_update;
-            layer.biases = &layer.biases + biases_update;
+
+            layer.set_weights(layer.get_weights() + weights_update);
+            layer.set_bias(layer.get_bias() + biases_update);
         }
     }
 
-    fn initialize(&mut self, layers: &Vec<Layer>) {
+    fn initialize(&mut self, layers: &Vec<Box<dyn Layer>>) {
         for layer in layers {
-            self.weights_cache.push(Array2::zeros(layer.weights.dim()));
-            self.biases_cache.push(Array2::zeros(layer.biases.dim()));
+            self.weights_cache
+                .push(Array2::zeros(layer.get_weights().dim()));
+            self.biases_cache
+                .push(Array2::zeros(layer.get_bias().dim()));
         }
     }
 

src/neural_network/optimizer/sgd_optimzer.rs

@@ -36,7 +36,7 @@ impl SGD {
 impl Optimizer for SGD {
     fn update_params(
         &mut self,
-        layers: &mut Vec<Layer>,
+        layers: &mut Vec<Box<dyn Layer>>,
         nabla_bs: &Vec<Array2<f64>>,
         nabla_ws: &Vec<Array2<f64>>,
     ) {
@@ -58,16 +58,17 @@ impl Optimizer for SGD {
             }
 
             //Update weights and biases
-            layer.weights = &layer.weights + weights_update;
-            layer.biases = &layer.biases + biases_update;
+            layer.set_weights(layer.get_weights() + weights_update);
+            layer.set_bias(layer.get_bias() + biases_update);
         }
     }
 
-    fn initialize(&mut self, layers: &Vec<Layer>) {
+    fn initialize(&mut self, layers: &Vec<Box<dyn Layer>>) {
         for layer in layers {
             self.weights_momentum
-                .push(Array2::zeros(layer.weights.dim()));
-            self.biases_momentum.push(Array2::zeros(layer.biases.dim()));
+                .push(Array2::zeros(layer.get_weights().dim()));
+            self.biases_momentum
+                .push(Array2::zeros(layer.get_bias().dim()));
         }
     }