From 50a7febe62522cc02e2a8847b067893944f49505 Mon Sep 17 00:00:00 2001 From: Ivan Date: Sun, 23 Jun 2019 18:54:53 +0200 Subject: [PATCH 1/3] Added network metadata to save and load subroutines --- src/lib/mod_layer.f90 | 9 ++++++++- src/lib/mod_network.f90 | 32 ++++++++++++++++++++++++-------- src/tests/test_network_save.f90 | 17 ++++++++++++++++- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/lib/mod_layer.f90 b/src/lib/mod_layer.f90 index 06d9a568..2bb70120 100644 --- a/src/lib/mod_layer.f90 +++ b/src/lib/mod_layer.f90 @@ -18,6 +18,7 @@ module mod_layer real(rk), allocatable :: z(:) ! arg. to activation function procedure(activation_function), pointer, nopass :: activation => null() procedure(activation_function), pointer, nopass :: activation_prime => null() + character(len=:), allocatable :: activation_str ! activation character string contains procedure, public, pass(self) :: set_activation end type layer_type @@ -115,7 +116,7 @@ subroutine dw_co_sum(dw) end do end subroutine dw_co_sum - pure subroutine set_activation(self, activation) + pure elemental subroutine set_activation(self, activation) ! Sets the activation function. Input string must match one of ! provided activation functions, otherwise it defaults to sigmoid. ! If activation not present, defaults to sigmoid. @@ -125,21 +126,27 @@ pure subroutine set_activation(self, activation) case('gaussian') self % activation => gaussian self % activation_prime => gaussian_prime + self % activation_str = 'gaussian' case('relu') self % activation => relu self % activation_prime => relu_prime + self % activation_str = 'relu' case('sigmoid') self % activation => sigmoid self % activation_prime => sigmoid_prime + self % activation_str = 'sigmoid' case('step') self % activation => step self % activation_prime => step_prime + self % activation_str = 'step' case('tanh') self % activation => tanhf self % activation_prime => tanh_prime + self % activation_str = 'tanh' case default self % activation => sigmoid self % activation_prime => sigmoid_prime + self % activation_str = 'sigmoid' end select end subroutine set_activation diff --git a/src/lib/mod_network.f90 b/src/lib/mod_network.f90 index 4d3d7aa0..44dfad72 100644 --- a/src/lib/mod_network.f90 +++ b/src/lib/mod_network.f90 @@ -25,12 +25,14 @@ module mod_network procedure, public, pass(self) :: loss procedure, public, pass(self) :: output procedure, public, pass(self) :: save - procedure, public, pass(self) :: set_activation + procedure, public, pass(self) :: set_activation_equal + procedure, public, pass(self) :: set_activation_layers procedure, public, pass(self) :: sync procedure, public, pass(self) :: train_batch procedure, public, pass(self) :: train_single procedure, public, pass(self) :: update + generic, public :: set_activation => set_activation_equal, set_activation_layers generic, public :: train => train_batch, train_single end type network_type @@ -136,13 +138,18 @@ subroutine load(self, filename) ! Loads the network from file. class(network_type), intent(in out) :: self character(len=*), intent(in) :: filename - integer(ik) :: fileunit, n, num_layers + integer(ik) :: fileunit, n, num_layers, layer_idx integer(ik), allocatable :: dims(:) + character(len=100) :: buffer ! activation string open(newunit=fileunit, file=filename, status='old', action='read') read(fileunit, fmt=*) num_layers allocate(dims(num_layers)) read(fileunit, fmt=*) dims call self % init(dims) + do n = 1, num_layers + read(fileunit, fmt=*) layer_idx, buffer + call self % layers(layer_idx) % set_activation(trim(buffer)) + end do do n = 2, size(self % dims) read(fileunit, fmt=*) self % layers(n) % b end do @@ -181,6 +188,9 @@ subroutine save(self, filename) open(newunit=fileunit, file=filename) write(fileunit, fmt=*) size(self % dims) write(fileunit, fmt=*) self % dims + do n = 1, size(self % dims) + write(fileunit, fmt=*) n, self % layers(n) % activation_str + end do do n = 2, size(self % dims) write(fileunit, fmt=*) self % layers(n) % b end do @@ -190,17 +200,23 @@ subroutine save(self, filename) close(fileunit) end subroutine save - pure subroutine set_activation(self, activation) + pure subroutine set_activation_equal(self, activation) ! A thin wrapper around layer % set_activation(). ! This method can be used to set an activation function ! for all layers at once. class(network_type), intent(in out) :: self character(len=*), intent(in) :: activation - integer :: n - do concurrent(n = 1:size(self % layers)) - call self % layers(n) % set_activation(activation) - end do - end subroutine set_activation + call self % layers(:) % set_activation(activation) + end subroutine set_activation_equal + + pure subroutine set_activation_layers(self, activation) + ! A thin wrapper around layer % set_activation(). + ! This method can be used to set different activation functions + ! for each layer separately. + class(network_type), intent(in out) :: self + character(len=*), intent(in) :: activation(size(self%layers)) + call self % layers(:) % set_activation(activation) + end subroutine set_activation_layers subroutine sync(self, image) ! Broadcasts network weights and biases from diff --git a/src/tests/test_network_save.f90 b/src/tests/test_network_save.f90 index 685961df..19114a10 100644 --- a/src/tests/test_network_save.f90 +++ b/src/tests/test_network_save.f90 @@ -6,6 +6,7 @@ program test_network_save print *, 'Initializing 2 networks with random weights and biases' net1 = network_type([768, 30, 10]) net2 = network_type([768, 30, 10]) + print *, 'Save network 1 into file' call net1 % save('test_network.dat') call net2 % load('test_network.dat') @@ -13,6 +14,20 @@ program test_network_save do n = 1, size(net1 % layers) print *, 'Layer ', n, ', weights equal: ',& all(net1 % layers(n) % w == net2 % layers(n) % w),& - ', biases equal:', all(net1 % layers(n) % b == net2 % layers(n) % b) + ', biases equal:', all(net1 % layers(n) % b == net2 % layers(n) % b),& + ', activation functions equal:', net1 % layers(n) % activation_str == net2 % layers(n) % activation_str end do + print *, '' + + print *, 'Setting different activation functions for each layer of network 1' + call net1 % set_activation([character(len=10) :: 'sigmoid', 'tanh', 'gaussian']) + print *, 'Save network 1 into file' + call net1 % save('test_network.dat') + call net2 % load('test_network.dat') + print *, 'Load network 2 from file' + do n = 1, size(net1 % layers) + print *, 'Layer ', n, ', activation functions equal:',& + net1 % layers(n) % activation_str == net2 % layers(n) % activation_str,& + '(network 1 - ', net1 % layers(n) % activation_str, ', network 2 - ', net2 % layers(n) % activation_str,')' + end do end program test_network_save From 1a723315583cef77a3ea43b73cd88b2c723128e7 Mon Sep 17 00:00:00 2001 From: Ivan Date: Sun, 23 Jun 2019 19:10:21 +0200 Subject: [PATCH 2/3] Modified test_network_save to test association with same procedure for activation functions --- src/tests/test_network_save.f90 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/tests/test_network_save.f90 b/src/tests/test_network_save.f90 index 19114a10..00aea0d2 100644 --- a/src/tests/test_network_save.f90 +++ b/src/tests/test_network_save.f90 @@ -14,8 +14,7 @@ program test_network_save do n = 1, size(net1 % layers) print *, 'Layer ', n, ', weights equal: ',& all(net1 % layers(n) % w == net2 % layers(n) % w),& - ', biases equal:', all(net1 % layers(n) % b == net2 % layers(n) % b),& - ', activation functions equal:', net1 % layers(n) % activation_str == net2 % layers(n) % activation_str + ', biases equal:', all(net1 % layers(n) % b == net2 % layers(n) % b) end do print *, '' @@ -27,7 +26,7 @@ program test_network_save print *, 'Load network 2 from file' do n = 1, size(net1 % layers) print *, 'Layer ', n, ', activation functions equal:',& - net1 % layers(n) % activation_str == net2 % layers(n) % activation_str,& - '(network 1 - ', net1 % layers(n) % activation_str, ', network 2 - ', net2 % layers(n) % activation_str,')' + associated(net1 % layers(n) % activation, net2 % layers(n) % activation),& + '(network 1: ', net1 % layers(n) % activation_str, ', network 2: ', net2 % layers(n) % activation_str,')' end do end program test_network_save From 35662125d6e6655092957554c7d2297506af670d Mon Sep 17 00:00:00 2001 From: milancurcic Date: Mon, 24 Jun 2019 07:00:35 -0400 Subject: [PATCH 3/3] style nit pick --- src/lib/mod_network.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mod_network.f90 b/src/lib/mod_network.f90 index 44dfad72..ed8451d3 100644 --- a/src/lib/mod_network.f90 +++ b/src/lib/mod_network.f90 @@ -214,7 +214,7 @@ pure subroutine set_activation_layers(self, activation) ! This method can be used to set different activation functions ! for each layer separately. class(network_type), intent(in out) :: self - character(len=*), intent(in) :: activation(size(self%layers)) + character(len=*), intent(in) :: activation(size(self % layers)) call self % layers(:) % set_activation(activation) end subroutine set_activation_layers