From 794c5266cee91188870fc062b22903323590baa0 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Sun, 30 Mar 2025 11:47:38 +0200 Subject: [PATCH 1/3] feat: add reverse cycle to inits components --- docs/src/api/inits.md | 1 + src/ReservoirComputing.jl | 4 +- src/esn/inits_components.jl | 118 ++++++++++++++++++++++++++++++++---- 3 files changed, 108 insertions(+), 15 deletions(-) diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index cca81e6f..618b0609 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -39,6 +39,7 @@ delay_line! backward_connection! simple_cycle! + reverse_simple_cycle! self_loop! add_jumps! ``` diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index ed8b6965..e6abbdc9 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -43,8 +43,8 @@ export rand_sparse, delay_line, delay_line_backward, cycle_jumps, simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, selfloop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, selfloop_forward_connection, forward_connection -export scale_radius!, delay_line!, backward_connection!, simple_cycle!, add_jumps!, - self_loop! +export scale_radius!, delay_line!, backward_connection!, simple_cycle!, reverse_simple_cycle!, + add_jumps!, self_loop! export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train export ESN, HybridESN, KnowledgeModel, DeepESN diff --git a/src/esn/inits_components.jl b/src/esn/inits_components.jl index a8172641..dd9dfc54 100644 --- a/src/esn/inits_components.jl +++ b/src/esn/inits_components.jl @@ -127,13 +127,17 @@ Adds a delay line in the `reservoir_matrix`, with given `shift` and If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each `weight` can be positive with a probability set by `positive_prob`. If set to `:irrational_sample!` the `weight` is negative if the decimal number of the - irrational number chosen is odd. Default is `:no_sample`. - - `positive_prob`: probability of the `weight` being positive with `sampling_type` - set to `:bernoulli_sample!`. Default is 0.5 + irrational number chosen is odd. If set to `:regular_sample!`, each weight will be + assigned a negative sign after the chosen `strides`. `strides` can be a single + number or an array. Default is `:no_sample`. + - `positive_prob`: probability of the `weight` being positive when `sampling_type` is + set to `:bernoulli_sample!`. Default is 0.5. - `irrational`: Irrational number whose decimals decide the sign of `weight`. Default is `pi`. - `start`: Which place after the decimal point the counting starts for the `irrational` sign counting. Default is 1. + - `strides`: number of strides for assigning negative value to a weight. It can be an + integer or an array. Default is 2. # Examples @@ -203,13 +207,17 @@ Adds a backward connection in the `reservoir_matrix`, with given `shift` and If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each `weight` can be positive with a probability set by `positive_prob`. If set to `:irrational_sample!` the `weight` is negative if the decimal number of the - irrational number chosen is odd. Default is `:no_sample`. - - `positive_prob`: probability of the `weight` being positive with `sampling_type` - set to `:bernoulli_sample!`. Default is 0.5 + irrational number chosen is odd. If set to `:regular_sample!`, each weight will be + assigned a negative sign after the chosen `strides`. `strides` can be a single + number or an array. Default is `:no_sample`. + - `positive_prob`: probability of the `weight` being positive when `sampling_type` is + set to `:bernoulli_sample!`. Default is 0.5. - `irrational`: Irrational number whose decimals decide the sign of `weight`. Default is `pi`. - `start`: Which place after the decimal point the counting starts for the `irrational` sign counting. Default is 1. + - `strides`: number of strides for assigning negative value to a weight. It can be an + integer or an array. Default is 2. # Examples @@ -279,13 +287,17 @@ Adds a simple cycle in the `reservoir_matrix`, with given If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each `weight` can be positive with a probability set by `positive_prob`. If set to `:irrational_sample!` the `weight` is negative if the decimal number of the - irrational number chosen is odd. Default is `:no_sample`. + irrational number chosen is odd. If set to `:regular_sample!`, each weight will be + assigned a negative sign after the chosen `strides`. `strides` can be a single + number or an array. Default is `:no_sample`. - `positive_prob`: probability of the `weight` being positive when `sampling_type` is - set to `:bernoulli_sample!`. Default is 0.5 + set to `:bernoulli_sample!`. Default is 0.5. - `irrational`: Irrational number whose decimals decide the sign of `weight`. Default is `pi`. - `start`: Which place after the decimal point the counting starts for the `irrational` sign counting. Default is 1. + - `strides`: number of strides for assigning negative value to a weight. It can be an + integer or an array. Default is 2. # Examples @@ -325,6 +337,78 @@ function simple_cycle!( return reservoir_matrix end +""" + reverse_simple_cycle!([rng], reservoir_matrix, weight; + sampling_type=:no_sample, irrational=pi, start=1, + p=0.5) + +Adds a reverse simple cycle in the `reservoir_matrix`, with given +`weight`. The `weight` can be a single number or an array. + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `reservoir_matrix`: matrix to be changed. + - `weight`: weight to add as a simple cycle. Can be either a single number + or an array. + +# Keyword arguments + + - `sampling_type`: Sampling that decides the distribution of `weight` negative numbers. + If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each + `weight` can be positive with a probability set by `positive_prob`. If set to + `:irrational_sample!` the `weight` is negative if the decimal number of the + irrational number chosen is odd. If set to `:regular_sample!`, each weight will be + assigned a negative sign after the chosen `strides`. `strides` can be a single + number or an array. Default is `:no_sample`. + - `positive_prob`: probability of the `weight` being positive when `sampling_type` is + set to `:bernoulli_sample!`. Default is 0.5. + - `irrational`: Irrational number whose decimals decide the sign of `weight`. + Default is `pi`. + - `start`: Which place after the decimal point the counting starts for the `irrational` + sign counting. Default is 1. + - `strides`: number of strides for assigning negative value to a weight. It can be an + integer or an array. Default is 2. + +# Examples + +```jldoctest +julia> matrix = zeros(Float32, 5, 5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 + +julia> reverse_simple_cycle!(matrix, 1.0; sampling_type=:regular_sample!) +5×5 Matrix{Float32}: + 0.0 -1.0 0.0 0.0 0.0 + 0.0 0.0 1.0 0.0 0.0 + 0.0 0.0 0.0 -1.0 0.0 + 0.0 0.0 0.0 0.0 1.0 + 1.0 0.0 0.0 0.0 0.0 +``` +""" +function reverse_simple_cycle!( + rng::AbstractRNG, reservoir_matrix::AbstractMatrix, weight::Number; kwargs...) + weights = fill(weight, size(reservoir_matrix, 1)) + return reverse_simple_cycle!(rng, reservoir_matrix, weights; kwargs...) +end + +function reverse_simple_cycle!( + rng::AbstractRNG, reservoir_matrix::AbstractMatrix, weight::AbstractVector; + sampling_type=:no_sample, kwargs...) + f_sample = getfield(@__MODULE__, sampling_type) + f_sample(rng, weight; kwargs...) + for idx in (first(axes(reservoir_matrix, 1)) + 1):last(axes(reservoir_matrix, 1)) + reservoir_matrix[idx - 1, idx] = weight[idx] + end + reservoir_matrix[end, 1] = weight[end] + return reservoir_matrix +end + """ add_jumps!([rng], reservoir_matrix, weight, jump_size; sampling_type=:no_sample, irrational=pi, start=1, @@ -348,13 +432,17 @@ Adds jumps to a given `reservoir_matrix` with chosen `weight` and determined `ju If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each `weight` can be positive with a probability set by `positive_prob`. If set to `:irrational_sample!` the `weight` is negative if the decimal number of the - irrational number chosen is odd. Default is `:no_sample`. + irrational number chosen is odd. If set to `:regular_sample!`, each weight will be + assigned a negative sign after the chosen `strides`. `strides` can be a single + number or an array. Default is `:no_sample`. - `positive_prob`: probability of the `weight` being positive when `sampling_type` is - set to `:bernoulli_sample!`. Default is 0.5 + set to `:bernoulli_sample!`. Default is 0.5. - `irrational`: Irrational number whose decimals decide the sign of `weight`. Default is `pi`. - `start`: Which place after the decimal point the counting starts for the `irrational` sign counting. Default is 1. + - `strides`: number of strides for assigning negative value to a weight. It can be an + integer or an array. Default is 2. # Examples @@ -423,13 +511,17 @@ Adds jumps to a given `reservoir_matrix` with chosen `weight` and determined `ju If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each `weight` can be positive with a probability set by `positive_prob`. If set to `:irrational_sample!` the `weight` is negative if the decimal number of the - irrational number chosen is odd. Default is `:no_sample`. + irrational number chosen is odd. If set to `:regular_sample!`, each weight will be + assigned a negative sign after the chosen `strides`. `strides` can be a single + number or an array. Default is `:no_sample`. - `positive_prob`: probability of the `weight` being positive when `sampling_type` is - set to `:bernoulli_sample!`. Default is 0.5 + set to `:bernoulli_sample!`. Default is 0.5. - `irrational`: Irrational number whose decimals decide the sign of `weight`. Default is `pi`. - `start`: Which place after the decimal point the counting starts for the `irrational` sign counting. Default is 1. + - `strides`: number of strides for assigning negative value to a weight. It can be an + integer or an array. Default is 2. # Examples @@ -468,7 +560,7 @@ function self_loop!(rng::AbstractRNG, reservoir_matrix::AbstractMatrix, end for init_component in (:delay_line!, :add_jumps!, :backward_connection!, - :simple_cycle!, :self_loop!) + :simple_cycle!, :reverse_simple_cycle!, :self_loop!) @eval begin function ($init_component)(args...; kwargs...) return $init_component(Utils.default_rng(), args...; kwargs...) From f1c5ec20ecf74d3ca45dc3a6c7752e4e19af2689 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Sun, 30 Mar 2025 12:00:47 +0200 Subject: [PATCH 2/3] feat(inits): add true_double_cycle reservoir initializer --- docs/src/api/inits.md | 1 + src/ReservoirComputing.jl | 5 ++- src/esn/esn_inits.jl | 77 ++++++++++++++++++++++++++++++++++++++- test/esn/test_inits.jl | 3 +- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index 618b0609..ac4209c6 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -25,6 +25,7 @@ simple_cycle cycle_jumps double_cycle + true_double_cycle selfloop_cycle selfloop_feedback_cycle selfloop_delayline_backward diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index e6abbdc9..af3703e5 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -42,8 +42,9 @@ export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mappin export rand_sparse, delay_line, delay_line_backward, cycle_jumps, simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, selfloop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, - selfloop_forward_connection, forward_connection -export scale_radius!, delay_line!, backward_connection!, simple_cycle!, reverse_simple_cycle!, + selfloop_forward_connection, forward_connection, true_double_cycle +export scale_radius!, delay_line!, backward_connection!, simple_cycle!, + reverse_simple_cycle!, add_jumps!, self_loop! export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train diff --git a/src/esn/esn_inits.jl b/src/esn/esn_inits.jl index a74b352a..682ad1e0 100644 --- a/src/esn/esn_inits.jl +++ b/src/esn/esn_inits.jl @@ -1432,6 +1432,81 @@ function double_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; return return_init_as(Val(return_sparse), reservoir_matrix) end +""" + true_double_cycle([rng], [T], dims...; + cycle_weight=0.1, second_cycle_weight=0.1, + return_sparse=false) + +Creates a true double cycle reservoir, ispired by [^fu2023], +with cycles built on the definition by [^rodan2010]. + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `cycle_weight`: Weight of the upper cycle connections in the reservoir matrix. + Default is 0.1. + + - `second_cycle_weight`: Weight of the lower cycle connections in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + - `cycle_kwargs`, and `second_cycle_kwargs`: named tuples that control the kwargs + for the weights generation. The kwargs are as follows: + + + `sampling_type`: Sampling that decides the distribution of `weight` negative numbers. + If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each + `weight` can be positive with a probability set by `positive_prob`. If set to + `:irrational_sample!` the `weight` is negative if the decimal number of the + irrational number chosen is odd. If set to `:regular_sample!`, each weight will be + assigned a negative sign after the chosen `strides`. `strides` can be a single + number or an array. Default is `:no_sample`. + + `positive_prob`: probability of the `weight` being positive when `sampling_type` is + set to `:bernoulli_sample!`. Default is 0.5. + + `irrational`: Irrational number whose decimals decide the sign of `weight`. + Default is `pi`. + + `start`: Which place after the decimal point the counting starts for the `irrational` + sign counting. Default is 1. + + `strides`: number of strides for assigning negative value to a weight. It can be an + integer or an array. Default is 2. + +# Examples + +```jldoctest +julia> true_double_cycle(5, 5; cycle_weight=0.1, second_cycle_weight=0.3) +5×5 Matrix{Float32}: + 0.0 0.3 0.0 0.0 0.1 + 0.1 0.0 0.3 0.0 0.0 + 0.0 0.1 0.0 0.3 0.0 + 0.0 0.0 0.1 0.0 0.3 + 0.3 0.0 0.0 0.1 0.0 +``` + +[^fu2023]: Fu, Jun, et al. + "A double-cycle echo state network topology for time series prediction." + Chaos: An Interdisciplinary Journal of Nonlinear Science 33.9 (2023). +[^rodan2010]: Rodan, Ali, and Peter Tino. + "Minimum complexity echo state network." + IEEE transactions on neural networks 22.1 (2010): 131-144. +""" +function true_double_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; + cycle_weight::Union{Number, AbstractVector}=T(0.1), + second_cycle_weight::Union{Number, AbstractVector}=T(0.1), + return_sparse::Bool=false, cycle_kwargs::NamedTuple=NamedTuple(), + second_cycle_kwargs::NamedTuple=NamedTuple()) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) + simple_cycle!(rng, reservoir_matrix, cycle_weight; cycle_kwargs...) + reverse_simple_cycle!( + rng, reservoir_matrix, second_cycle_weight; second_cycle_kwargs...) + return return_init_as(Val(return_sparse), reservoir_matrix) +end + @doc raw""" selfloop_cycle([rng], [T], dims...; cycle_weight=0.1, selfloop_weight=0.1, @@ -1908,7 +1983,7 @@ for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jump :weighted_minimal, :informed_init, :minimal_init, :chebyshev_mapping, :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :selfloop_cycle, :selfloop_feedback_cycle, :selfloop_delayline_backward, :selfloop_forward_connection, - :forward_connection) + :forward_connection, :true_double_cycle) @eval begin function ($initializer)(dims::Integer...; kwargs...) return $initializer(Utils.default_rng(), Float32, dims...; kwargs...) diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl index 8bb43f63..29095473 100644 --- a/test/esn/test_inits.jl +++ b/test/esn/test_inits.jl @@ -32,7 +32,8 @@ reservoir_inits = [ selfloop_feedback_cycle, selfloop_delayline_backward, selfloop_forward_connection, - forward_connection + forward_connection, + true_double_cycle ] input_inits = [ scaled_rand, From 2b55a6ee295c3c7eb5f6f0682a89feea16dae187 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Sun, 30 Mar 2025 12:09:28 +0200 Subject: [PATCH 3/3] chore: up version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 1dbaab94..f788bec7 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ReservoirComputing" uuid = "7c2d2b1e-3dd4-11ea-355a-8f6a8116e294" authors = ["Francesco Martinuzzi"] -version = "0.11.1" +version = "0.11.2" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"