feat(inits): add true_double_cycle reservoir initializer

Project.toml

@@ -1,7 +1,7 @@
 name = "ReservoirComputing"
 uuid = "7c2d2b1e-3dd4-11ea-355a-8f6a8116e294"
 authors = ["Francesco Martinuzzi"]
-version = "0.11.1"
+version = "0.11.2"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"

docs/src/api/inits.md

@@ -25,6 +25,7 @@
     simple_cycle
     cycle_jumps
     double_cycle
+    true_double_cycle
     selfloop_cycle
     selfloop_feedback_cycle
     selfloop_delayline_backward
@@ -39,6 +40,7 @@
     delay_line!
     backward_connection!
     simple_cycle!
+    reverse_simple_cycle!
     self_loop!
     add_jumps!
 ```

src/ReservoirComputing.jl

@@ -42,9 +42,10 @@ export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mappin
 export rand_sparse, delay_line, delay_line_backward, cycle_jumps,
        simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle,
        selfloop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward,
-       selfloop_forward_connection, forward_connection
-export scale_radius!, delay_line!, backward_connection!, simple_cycle!, add_jumps!,
-       self_loop!
+       selfloop_forward_connection, forward_connection, true_double_cycle
+export scale_radius!, delay_line!, backward_connection!, simple_cycle!,
+       reverse_simple_cycle!,
+       add_jumps!, self_loop!
 export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal
 export train
 export ESN, HybridESN, KnowledgeModel, DeepESN

src/esn/esn_inits.jl

@@ -1432,6 +1432,81 @@ function double_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...;
     return return_init_as(Val(return_sparse), reservoir_matrix)
 end
 
+"""
+    true_double_cycle([rng], [T], dims...; 
+        cycle_weight=0.1, second_cycle_weight=0.1,
+        return_sparse=false)
+
+Creates a true double cycle reservoir, ispired by [^fu2023],
+with cycles built on the definition by [^rodan2010].
+
+# Arguments
+
+  - `rng`: Random number generator. Default is `Utils.default_rng()`
+    from WeightInitializers.
+  - `T`: Type of the elements in the reservoir matrix. Default is `Float32`.
+  - `dims`: Dimensions of the reservoir matrix.
+
+# Keyword arguments
+
+  - `cycle_weight`: Weight of the upper cycle connections in the reservoir matrix.
+    Default is 0.1.
+
+  - `second_cycle_weight`: Weight of the lower cycle connections in the reservoir matrix.
+    Default is 0.1.
+  - `return_sparse`: flag for returning a `sparse` matrix.
+    Default is `false`.
+  - `cycle_kwargs`, and `second_cycle_kwargs`: named tuples that control the kwargs
+    for the weights generation. The kwargs are as follows:
+
+      + `sampling_type`: Sampling that decides the distribution of `weight` negative numbers.
+        If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
+        `weight` can be positive with a probability set by `positive_prob`. If set to
+        `:irrational_sample!` the `weight` is negative if the decimal number of the
+        irrational number chosen is odd. If set to `:regular_sample!`, each weight will be
+        assigned a negative sign after the chosen `strides`. `strides` can be a single
+        number or an array. Default is `:no_sample`.
+      + `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+        set to `:bernoulli_sample!`. Default is 0.5.
+      + `irrational`: Irrational number whose decimals decide the sign of `weight`.
+        Default is `pi`.
+      + `start`: Which place after the decimal point the counting starts for the `irrational`
+        sign counting. Default is 1.
+      + `strides`: number of strides for assigning negative value to a weight. It can be an
+        integer or an array. Default is 2.
+
+# Examples
+
+```jldoctest
+julia> true_double_cycle(5, 5; cycle_weight=0.1, second_cycle_weight=0.3)
+5×5 Matrix{Float32}:
+ 0.0  0.3  0.0  0.0  0.1
+ 0.1  0.0  0.3  0.0  0.0
+ 0.0  0.1  0.0  0.3  0.0
+ 0.0  0.0  0.1  0.0  0.3
+ 0.3  0.0  0.0  0.1  0.0
+```
+
+[^fu2023]: Fu, Jun, et al.
+    "A double-cycle echo state network topology for time series prediction."
+    Chaos: An Interdisciplinary Journal of Nonlinear Science 33.9 (2023).
+[^rodan2010]: Rodan, Ali, and Peter Tino.
+    "Minimum complexity echo state network."
+    IEEE transactions on neural networks 22.1 (2010): 131-144.
+"""
+function true_double_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...;
+        cycle_weight::Union{Number, AbstractVector}=T(0.1),
+        second_cycle_weight::Union{Number, AbstractVector}=T(0.1),
+        return_sparse::Bool=false, cycle_kwargs::NamedTuple=NamedTuple(),
+        second_cycle_kwargs::NamedTuple=NamedTuple()) where {T <: Number}
+    throw_sparse_error(return_sparse)
+    reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...)
+    simple_cycle!(rng, reservoir_matrix, cycle_weight; cycle_kwargs...)
+    reverse_simple_cycle!(
+        rng, reservoir_matrix, second_cycle_weight; second_cycle_kwargs...)
+    return return_init_as(Val(return_sparse), reservoir_matrix)
+end
+
 @doc raw"""
     selfloop_cycle([rng], [T], dims...; 
         cycle_weight=0.1, selfloop_weight=0.1,
@@ -1908,7 +1983,7 @@ for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jump
     :weighted_minimal, :informed_init, :minimal_init, :chebyshev_mapping,
     :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :selfloop_cycle,
     :selfloop_feedback_cycle, :selfloop_delayline_backward, :selfloop_forward_connection,
-    :forward_connection)
+    :forward_connection, :true_double_cycle)
     @eval begin
         function ($initializer)(dims::Integer...; kwargs...)
             return $initializer(Utils.default_rng(), Float32, dims...; kwargs...)

src/esn/inits_components.jl

@@ -127,13 +127,17 @@ Adds a delay line in the `reservoir_matrix`, with given `shift` and
     If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
     `weight` can be positive with a probability set by `positive_prob`. If set to
     `:irrational_sample!` the `weight` is negative if the decimal number of the
-    irrational number chosen is odd. Default is `:no_sample`.
-  - `positive_prob`: probability of the `weight` being positive with `sampling_type`
-    set to `:bernoulli_sample!`. Default is 0.5
+    irrational number chosen is odd. If set to `:regular_sample!`, each weight will be
+    assigned a negative sign after the chosen `strides`. `strides` can be a single
+    number or an array. Default is `:no_sample`.
+  - `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+    set to `:bernoulli_sample!`. Default is 0.5.
   - `irrational`: Irrational number whose decimals decide the sign of `weight`.
     Default is `pi`.
   - `start`: Which place after the decimal point the counting starts for the `irrational`
     sign counting. Default is 1.
+  - `strides`: number of strides for assigning negative value to a weight. It can be an
+    integer or an array. Default is 2.
 
 # Examples
 
@@ -203,13 +207,17 @@ Adds a backward connection in the `reservoir_matrix`, with given `shift` and
     If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
     `weight` can be positive with a probability set by `positive_prob`. If set to
     `:irrational_sample!` the `weight` is negative if the decimal number of the
-    irrational number chosen is odd. Default is `:no_sample`.
-  - `positive_prob`: probability of the `weight` being positive with `sampling_type`
-    set to `:bernoulli_sample!`. Default is 0.5
+    irrational number chosen is odd. If set to `:regular_sample!`, each weight will be
+    assigned a negative sign after the chosen `strides`. `strides` can be a single
+    number or an array. Default is `:no_sample`.
+  - `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+    set to `:bernoulli_sample!`. Default is 0.5.
   - `irrational`: Irrational number whose decimals decide the sign of `weight`.
     Default is `pi`.
   - `start`: Which place after the decimal point the counting starts for the `irrational`
     sign counting. Default is 1.
+  - `strides`: number of strides for assigning negative value to a weight. It can be an
+    integer or an array. Default is 2.
 
 # Examples
 
@@ -279,13 +287,17 @@ Adds a simple cycle in the `reservoir_matrix`, with given
     If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
     `weight` can be positive with a probability set by `positive_prob`. If set to
     `:irrational_sample!` the `weight` is negative if the decimal number of the
-    irrational number chosen is odd. Default is `:no_sample`.
+    irrational number chosen is odd. If set to `:regular_sample!`, each weight will be
+    assigned a negative sign after the chosen `strides`. `strides` can be a single
+    number or an array. Default is `:no_sample`.
   - `positive_prob`: probability of the `weight` being positive when `sampling_type` is
-    set to `:bernoulli_sample!`. Default is 0.5
+    set to `:bernoulli_sample!`. Default is 0.5.
   - `irrational`: Irrational number whose decimals decide the sign of `weight`.
     Default is `pi`.
   - `start`: Which place after the decimal point the counting starts for the `irrational`
     sign counting. Default is 1.
+  - `strides`: number of strides for assigning negative value to a weight. It can be an
+    integer or an array. Default is 2.
 
 # Examples
 
@@ -325,6 +337,78 @@ function simple_cycle!(
     return reservoir_matrix
 end
 
+"""
+    reverse_simple_cycle!([rng], reservoir_matrix, weight;
+        sampling_type=:no_sample, irrational=pi, start=1,
+        p=0.5)
+
+Adds a reverse simple cycle in the `reservoir_matrix`, with given
+`weight`. The `weight` can be a single number or an array.
+
+# Arguments
+
+  - `rng`: Random number generator. Default is `Utils.default_rng()`
+    from WeightInitializers.
+  - `reservoir_matrix`: matrix to be changed.
+  - `weight`: weight to add as a simple cycle. Can be either a single number
+    or an array.
+
+# Keyword arguments
+
+  - `sampling_type`: Sampling that decides the distribution of `weight` negative numbers.
+    If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
+    `weight` can be positive with a probability set by `positive_prob`. If set to
+    `:irrational_sample!` the `weight` is negative if the decimal number of the
+    irrational number chosen is odd. If set to `:regular_sample!`, each weight will be
+    assigned a negative sign after the chosen `strides`. `strides` can be a single
+    number or an array. Default is `:no_sample`.
+  - `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+    set to `:bernoulli_sample!`. Default is 0.5.
+  - `irrational`: Irrational number whose decimals decide the sign of `weight`.
+    Default is `pi`.
+  - `start`: Which place after the decimal point the counting starts for the `irrational`
+    sign counting. Default is 1.
+  - `strides`: number of strides for assigning negative value to a weight. It can be an
+    integer or an array. Default is 2.
+
+# Examples
+
+```jldoctest
+julia> matrix = zeros(Float32, 5, 5)
+5×5 Matrix{Float32}:
+ 0.0  0.0  0.0  0.0  0.0
+ 0.0  0.0  0.0  0.0  0.0
+ 0.0  0.0  0.0  0.0  0.0
+ 0.0  0.0  0.0  0.0  0.0
+ 0.0  0.0  0.0  0.0  0.0
+
+julia> reverse_simple_cycle!(matrix, 1.0; sampling_type=:regular_sample!)
+5×5 Matrix{Float32}:
+ 0.0  -1.0  0.0   0.0  0.0
+ 0.0   0.0  1.0   0.0  0.0
+ 0.0   0.0  0.0  -1.0  0.0
+ 0.0   0.0  0.0   0.0  1.0
+ 1.0   0.0  0.0   0.0  0.0
+```
+"""
+function reverse_simple_cycle!(
+        rng::AbstractRNG, reservoir_matrix::AbstractMatrix, weight::Number; kwargs...)
+    weights = fill(weight, size(reservoir_matrix, 1))
+    return reverse_simple_cycle!(rng, reservoir_matrix, weights; kwargs...)
+end
+
+function reverse_simple_cycle!(
+        rng::AbstractRNG, reservoir_matrix::AbstractMatrix, weight::AbstractVector;
+        sampling_type=:no_sample, kwargs...)
+    f_sample = getfield(@__MODULE__, sampling_type)
+    f_sample(rng, weight; kwargs...)
+    for idx in (first(axes(reservoir_matrix, 1)) + 1):last(axes(reservoir_matrix, 1))
+        reservoir_matrix[idx - 1, idx] = weight[idx]
+    end
+    reservoir_matrix[end, 1] = weight[end]
+    return reservoir_matrix
+end
+
 """
     add_jumps!([rng], reservoir_matrix, weight, jump_size;
         sampling_type=:no_sample, irrational=pi, start=1,
@@ -348,13 +432,17 @@ Adds jumps to a given `reservoir_matrix` with chosen `weight` and determined `ju
     If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
     `weight` can be positive with a probability set by `positive_prob`. If set to
     `:irrational_sample!` the `weight` is negative if the decimal number of the
-    irrational number chosen is odd. Default is `:no_sample`.
+    irrational number chosen is odd. If set to `:regular_sample!`, each weight will be
+    assigned a negative sign after the chosen `strides`. `strides` can be a single
+    number or an array. Default is `:no_sample`.
   - `positive_prob`: probability of the `weight` being positive when `sampling_type` is
-    set to `:bernoulli_sample!`. Default is 0.5
+    set to `:bernoulli_sample!`. Default is 0.5.
   - `irrational`: Irrational number whose decimals decide the sign of `weight`.
     Default is `pi`.
   - `start`: Which place after the decimal point the counting starts for the `irrational`
     sign counting. Default is 1.
+  - `strides`: number of strides for assigning negative value to a weight. It can be an
+    integer or an array. Default is 2.
 
 # Examples
 
@@ -423,13 +511,17 @@ Adds jumps to a given `reservoir_matrix` with chosen `weight` and determined `ju
     If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
     `weight` can be positive with a probability set by `positive_prob`. If set to
     `:irrational_sample!` the `weight` is negative if the decimal number of the
-    irrational number chosen is odd. Default is `:no_sample`.
+    irrational number chosen is odd. If set to `:regular_sample!`, each weight will be
+    assigned a negative sign after the chosen `strides`. `strides` can be a single
+    number or an array. Default is `:no_sample`.
   - `positive_prob`: probability of the `weight` being positive when `sampling_type` is
-    set to `:bernoulli_sample!`. Default is 0.5
+    set to `:bernoulli_sample!`. Default is 0.5.
   - `irrational`: Irrational number whose decimals decide the sign of `weight`.
     Default is `pi`.
   - `start`: Which place after the decimal point the counting starts for the `irrational`
     sign counting. Default is 1.
+  - `strides`: number of strides for assigning negative value to a weight. It can be an
+    integer or an array. Default is 2.
 
 # Examples
 
@@ -468,7 +560,7 @@ function self_loop!(rng::AbstractRNG, reservoir_matrix::AbstractMatrix,
 end
 
 for init_component in (:delay_line!, :add_jumps!, :backward_connection!,
-    :simple_cycle!, :self_loop!)
+    :simple_cycle!, :reverse_simple_cycle!, :self_loop!)
     @eval begin
         function ($init_component)(args...; kwargs...)
             return $init_component(Utils.default_rng(), args...; kwargs...)

test/esn/test_inits.jl

@@ -32,7 +32,8 @@ reservoir_inits = [
     selfloop_feedback_cycle,
     selfloop_delayline_backward,
     selfloop_forward_connection,
-    forward_connection
+    forward_connection,
+    true_double_cycle
 ]
 input_inits = [
     scaled_rand,