From 2399af1723aa3bc13fc05312d1484efc7e5759ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sat, 11 Jan 2025 09:39:37 +1100 Subject: [PATCH 01/41] change signature for reg. - add dyn selection method test green --- src/Gep.jl | 10 +++++---- src/Sbp.jl | 2 +- src/Selection.jl | 47 +++++++++++++++++++-------------------- test/Spb_core_test.jl | 6 ++--- test/non_dom_sort_test.jl | 19 ++++++++-------- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/Gep.jl b/src/Gep.jl index 26fd865..571240b 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -284,7 +284,8 @@ function runGep(epochs::Int, correction_amount::Real=0.6, tourni_size::Int=3, opt_method_const::Symbol=:cg, - optimisation_epochs::Int=500) where {T<:AbstractFloat} + optimisation_epochs::Int=500, + selection_mechanism::Function=basic_tournament_selection) where {T<:AbstractFloat} loss_fun = typeof(loss_fun_) == String ? get_loss_function(loss_fun_) : loss_fun_ @@ -349,10 +350,11 @@ function runGep(epochs::Int, if isclose(fits_representation[1], zero(T)) break end - + if epoch < epochs - indices = basic_tournament_selection(fits_representation[1:mating_size], tourni_size, mating_size) - parents = population[indices] + #needs to be adapted for multi objective -> how to return two + selectedMembers = selection_mechanism(fits_representation[1:mating_size], mating_size, tourni_size) + parents = population[selectedMembers.indices] perform_step!(population, parents, next_gen, toolbox, mating_size) end diff --git a/src/Sbp.jl b/src/Sbp.jl index 2e8b490..e68b852 100644 --- a/src/Sbp.jl +++ b/src/Sbp.jl @@ -1014,7 +1014,7 @@ function propagate_necessary_changes!( return true end - if check_crit_up!(tree.depend_on_total_number+1, expected_dim, tree) && distance_to_change <= 0 && rand() > 0.25 + if check_crit_up!(tree.depend_on_total_number+1, expected_dim, tree) && distance_to_change <= 0 && rand() > 0.05 return enforce_changes!(tree, expected_dim) end diff --git a/src/Selection.jl b/src/Selection.jl index 82895f7..0300009 100644 --- a/src/Selection.jl +++ b/src/Selection.jl @@ -4,10 +4,14 @@ using LinearAlgebra export selection_NSGA, basic_tournament_selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance +struct SelectedMembers + indices::Vector{Int} + fronts::Dict{Int,Vector{Int}} +end #Note: selection is constructed to allways return a list of indices => {just care about the data not the objects} -@inline function basic_tournament_selection(population::AbstractArray{T}, tournament_size::Int, number_of_winners::Int) where {T<:Number} +@inline function basic_tournament_selection(population::AbstractArray{T}, number_of_winners::Int, tournament_size::Int) where {T<:Number} selected_indices = Vector{Int}(undef, number_of_winners) valid_indices_ = findall(isfinite, population) valid_indices = [] @@ -25,7 +29,7 @@ export selection_NSGA, basic_tournament_selection, dominates_, fast_non_dominate selected_indices[index] = winner end selected_indices[end] = 1 - return selected_indices + return SelectedMembers(selected_indices,Dict{Int,Vector{Int}}()) end @@ -161,38 +165,33 @@ end return distances end -@inline function selection_NSGA(population::Vector{T}, num_to_select::Int) where {T<:Tuple} +@inline function selection_NSGA(population::Vector{T}, number_of_winners::Int, tournament_size::Int) where {T<:Tuple} fronts = calculate_fronts(population) n_fronts = length(fronts) selected_indices = Int[] pareto_fronts = Dict{Int,Vector{Int}}() - for front_idx in 1:n_fronts - front = fronts[front_idx] - pareto_fronts[front_idx] = front - - if length(selected_indices) + length(front) <= num_to_select - append!(selected_indices, front) - else - remaining_slots = num_to_select - length(selected_indices) - crowding_distances = assign_crowding_distance(front, population) + estimated_size = length(population) + selected_indices = Vector{Int}(undef, estimated_size) + current_idx = 1 - sorted_front = sort(front, by=i -> crowding_distances[i], rev=true) - - append!(selected_indices, sorted_front[1:remaining_slots]) - break - end - if length(selected_indices) >= num_to_select - break - end + @inbounds for front_idx in 1:n_fronts + front = fronts[front_idx] + crowding_distances = assign_crowding_distance(front, population) + + sorted_front = sort(front, by=i -> crowding_distances[i], rev=true) + front_size = length(sorted_front) + copyto!(selected_indices, current_idx, sorted_front, 1, front_size) + + pareto_fronts[front_idx] = sorted_front + current_idx +=front_size end + resize!(selected_indices, current_idx - 1) - if length(selected_indices) < num_to_select - @warn "Not enough individuals in the population to select $(num_to_select). Returning $(length(selected_indices)) individuals." - end + return SelectedMembers(selected_indices, pareto_fronts) - return selected_indices, pareto_fronts end + end diff --git a/test/Spb_core_test.jl b/test/Spb_core_test.jl index 8538a12..f044909 100644 --- a/test/Spb_core_test.jl +++ b/test/Spb_core_test.jl @@ -6,7 +6,7 @@ include("../src/Util.jl") using .SBPUtils using Random using OrderedCollections - +Random.seed!(1) function create_token_lib_test() physical_dimension_dict = OrderedDict{Int8, Vector{Float16}}( @@ -52,7 +52,7 @@ end features = Int8[7, 8] # x1, x2 functions = Int8[1, 2, 3, 4, 5, 6] # mul, div, add, sub, sqr, sin constants = Int8[] - lib = create_lib(token_lib, features, functions, constants; rounds=2, max_permutations=10000) + lib = create_lib(token_lib, features, functions, constants; rounds=8, max_permutations=10000) total_len_lib = sum(length(entry) for entry in values(lib)) @show ("Lib Entries:" , total_len_lib) @test !isempty(lib) @@ -106,7 +106,7 @@ end features = Int8[7, 8] # x1, x2 functions = Int8[1, 2, 3, 4, 5, 6] # mul, div, add, sub, sqr, sin constants = Int8[] - lib = create_lib(token_lib, features, functions, constants; rounds=6, max_permutations=10000) + lib = create_lib(token_lib, features, functions, constants; rounds=8, max_permutations=100000) point_operations = Int8[1,2] inverse_operation = Dict{Int8, Function}( 1 => (mul_unit_backward), diff --git a/test/non_dom_sort_test.jl b/test/non_dom_sort_test.jl index 09a425b..a560528 100644 --- a/test/non_dom_sort_test.jl +++ b/test/non_dom_sort_test.jl @@ -55,7 +55,6 @@ end ]) fronts3 = calculate_fronts(pop3) - @show fronts3 @test length(fronts3) == 3 @test Set(fronts3[1]) == Set([1]) @test Set(fronts3[2]) == Set([2, 4, 5, 6, 7, 8, 9]) @@ -90,10 +89,10 @@ end (1, 2), (2, 1), (1, 1) ]) - selected2, fronts2 = selection_NSGA(pop2, 10) - @test length(selected2) == 10 - @test 15 in selected2 # Preserve the best!! alllllways - @test length(fronts2) == 4 + selected2 = selection_NSGA(pop2, 10, 0) + @test length(selected2.indices) == 15 + @test 15 in selected2.indices # Preserve the best!! alllllways + @test length(selected2.fronts) == 5 # 3 objectives pop3 = create_population([ @@ -101,8 +100,8 @@ end (1, 2, 3), (2, 3, 1), (3, 1, 2), (1, 3, 2), (2, 1, 3), (3, 2, 1) ]) - selected3, fronts3 = selection_NSGA(pop3, 5) - @test length(selected3) == 5 - @test 1 in selected3 # The best individual should always be selected - @test length(fronts3) == 2 -end \ No newline at end of file + selected3 = selection_NSGA(pop3, 5, 0) + @test length(selected3.indices) == 9 + @test 1 in selected3.indices # The best individual should always be selected + @test length(selected3.fronts) == 3 +end From 484d0b382c7b51e1d7f2bacfeec5e03bd753d6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sat, 11 Jan 2025 10:00:58 +1100 Subject: [PATCH 02/41] del uness consideration --- src/Entities.jl | 3 --- src/Gep.jl | 39 +++++++++++++++++++++++++++++---------- src/RegressionWrapper.jl | 4 +--- test/Main_min_example.jl | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index a621cc1..f883c5b 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -169,7 +169,6 @@ Represents an individual solution in GEP. - `fitness_r2_test::AbstractFloat`: R² score on testing - `expression_raw::Vector{Int8}`: Raw expression - `dimension_homogene::Bool`: Dimensional homogeneity -- `penalty::AbstractFloat`: Penalty value - `chromo_id::Int`: Chromosome identifier # Constructor @@ -185,7 +184,6 @@ mutable struct Chromosome fitness_r2_test::AbstractFloat expression_raw::Vector{Int8} dimension_homogene::Bool - penalty::AbstractFloat chromo_id::Int function Chromosome(genes::Vector{Int8}, toolbox::Toolbox, compile::Bool=false) @@ -198,7 +196,6 @@ mutable struct Chromosome obj.fitness_r2_test = 0.0 obj.compiled = false obj.dimension_homogene = false - obj.penalty = 0.0 obj.chromo_id = -1 if compile compile_expression!(obj) diff --git a/src/Gep.jl b/src/Gep.jl index 571240b..dd3727b 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -93,7 +93,7 @@ const Toolbox = GepEntities.Toolbox """ compute_fitness(elem::Chromosome, operators::OperatorEnum, x_data::AbstractArray{T}, y_data::AbstractArray{T}, loss_function::Function, crash_value::T; - validate::Bool=false, penalty_consideration::Real=0.0) where {T<:AbstractFloat} + validate::Bool=false) where {T<:AbstractFloat} Computes the fitness score for a chromosome using the specified loss function. @@ -105,24 +105,22 @@ Computes the fitness score for a chromosome using the specified loss function. - `loss_function::Function`: The loss function used to compute fitness - `crash_value::T`: Default value returned if computation fails - `validate::Bool=false`: If true, forces recomputation of fitness even if already calculated -- `penalty_consideration::Real=0.0`: Additional penalty term added to the fitness score # Returns -Returns the computed fitness value (loss + penalty) or crash_value if computation fails +Returns the computed fitness value (loss) or crash_value if computation fails # Details - Checks if fitness needs to be computed (if NaN or validate=true) - Evaluates the chromosome's compiled function on input data -- Applies loss function and adds any penalty consideration - Returns crash_value if any errors occur during computation """ @inline function compute_fitness(elem::Chromosome, operators::OperatorEnum, x_data::AbstractArray{T}, y_data::AbstractArray{T}, loss_function::Function, - crash_value::T; validate::Bool=false, penalty_consideration::Real=0.0) where {T<:AbstractFloat} + crash_value::T; validate::Bool=false) where {T<:AbstractFloat} try if isnan(elem.fitness) || validate y_pred = elem.compiled_function(x_data, operators) - return loss_function(y_data, y_pred) + penalty_consideration + return loss_function(y_data, y_pred) else return elem.fitness end @@ -296,7 +294,6 @@ function runGep(epochs::Int, return get_loss_function("mse")(y_pred, y_data) end - penalty_consideration = convert(Real,toolbox.gep_probs["penalty_consideration"]) mating_ = toolbox.gep_probs["mating_size"] mating_size = Int(ceil(population_size * mating_)) mating_size = mating_size % 2 == 0 ? mating_size : mating_size - 1 @@ -313,8 +310,7 @@ function runGep(epochs::Int, Threads.@threads for i in eachindex(population) if isnan(population[i].fitness) - population[i].fitness = compute_fitness(population[i], operators, x_data, y_data, loss_fun, typemax(T); - penalty_consideration=population[i].penalty * penalty_consideration) + population[i].fitness = compute_fitness(population[i], operators, x_data, y_data, loss_fun, typemax(T)) end end @@ -351,8 +347,8 @@ function runGep(epochs::Int, break end + #ref -> done - dyn select method if epoch < epochs - #needs to be adapted for multi objective -> how to return two selectedMembers = selection_mechanism(fits_representation[1:mating_size], mating_size, tourni_size) parents = population[selectedMembers.indices] perform_step!(population, parents, next_gen, toolbox, mating_size) @@ -371,4 +367,27 @@ function runGep(epochs::Int, close_recorder!(recorder) return best, recorder.history end + + + + + + + + + + + + + + + + + + + + + + + end diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index c31567b..0b99c8c 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -363,7 +363,6 @@ Dictionary containing default probabilities and parameters for genetic algorithm - `fusion_rate`: Rate of general fusion (0.0) - `inversion_prob`: Probability of inversion (0.1) - `mating_size`: Relative size of mating pool (0.5) -- `penalty_consideration`: Weight of penalty in fitness evaluation (0.2) These values can be adjusted to fine-tune the genetic algorithm's behavior. """ @@ -381,8 +380,7 @@ const GENE_COMMON_PROBS = Dict{String,AbstractFloat}( "inversion_prob" => 0.1, "reverse_insertion" => 0.05, "reverse_insertion_tail" => 0.05, - "mating_size" => 0.5, - "penalty_consideration" => 0.2) + "mating_size" => 0.5) const SymbolDict = OrderedDict{Int8,Int8} const CallbackDict = Dict{Int8,Function} diff --git a/test/Main_min_example.jl b/test/Main_min_example.jl index 34017a3..3983e14 100644 --- a/test/Main_min_example.jl +++ b/test/Main_min_example.jl @@ -14,7 +14,7 @@ population_size = 1000 number_features = 2 x_data = randn(Float64, 100, number_features) -y_data = @. x_data[:,1] * x_data[:,1] + x_data[:,1] * x_data[:,2] - 2 * x_data[:,1] * x_data[:,2] +y_data = @. x_data[:,1] * x_data[:,1] + x_data[:,1] * x_data[:,2] - 2 * x_data[:,2] * x_data[:,2] #define the regressor = GepRegressor(number_features) From 0fa162e4e0098fd72f62cf34dcf71d34d7ffd1b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sat, 11 Jan 2025 13:48:44 +1100 Subject: [PATCH 03/41] adding strategies --- src/Gep.jl | 101 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 91 insertions(+), 10 deletions(-) diff --git a/src/Gep.jl b/src/Gep.jl index dd3727b..b19609c 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -57,7 +57,7 @@ See also: - [`GepEntities.fitness`](@ref): Fitness value access - [`GepEntities.set_fitness!`](@ref): Fitness value modification - +#TODO need to create different strategies for wrapping costum functions """ module GepRegression @@ -89,7 +89,24 @@ using Printf const Chromosome = GepEntities.Chromosome const Toolbox = GepEntities.Toolbox +abstract type EvaluationStrategy{R} end + +struct StandardRegression{T<:AbstractFloat} <: EvaluationStrategy{T} + operators::OperatorEnum + x_data::AbstractArray{T} + y_data::AbstractArray{T} + evaluator::Function + crash_value::T +end + +struct GenerEvaluation{R} <: EvaluationStrategy{R} + operators::OperatorEnum + evaluator::Function + crash_value::R +end + +#redesign -> compute fitness should return fitness and crash, we just need to insert the chromosome """ compute_fitness(elem::Chromosome, operators::OperatorEnum, x_data::AbstractArray{T}, y_data::AbstractArray{T}, loss_function::Function, crash_value::T; @@ -283,21 +300,22 @@ function runGep(epochs::Int, tourni_size::Int=3, opt_method_const::Symbol=:cg, optimisation_epochs::Int=500, - selection_mechanism::Function=basic_tournament_selection) where {T<:AbstractFloat} + selection_mechanism::Function=basic_tournament_selection, + num_objectives::Int=1) where {T<:AbstractFloat} loss_fun = typeof(loss_fun_) == String ? get_loss_function(loss_fun_) : loss_fun_ recorder = HistoryRecorder(epochs, Float64) function optimizer_function(sub_tree::Node) - y_pred, flag = eval_tree_array(sub_tree, x_data, operators) + y_pred, flag = eval_tree_array(sub_tree, x_data, operators) #another form of the compute fitness return get_loss_function("mse")(y_pred, y_data) end mating_ = toolbox.gep_probs["mating_size"] mating_size = Int(ceil(population_size * mating_)) mating_size = mating_size % 2 == 0 ? mating_size : mating_size - 1 - fits_representation = Vector{T}(undef, population_size) + fits_representation = num_objectives == 1 ? Vector{T}(undef, population_size) : Vector{Tuple}(undef, population_size) population = generate_population(population_size, toolbox) next_gen = Vector{eltype(population)}(undef, mating_size) @@ -309,12 +327,12 @@ function runGep(epochs::Int, perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) Threads.@threads for i in eachindex(population) - if isnan(population[i].fitness) + if isnan(mean(population[i].fitness)) population[i].fitness = compute_fitness(population[i], operators, x_data, y_data, loss_fun, typemax(T)) end end - sort!(population, by=x -> x.fitness) + sort!(population, by=x -> mean(x.fitness)) try if (prev_best == -1 || prev_best > population[1].fitness) && epoch % optimisation_epochs == 0 @@ -368,26 +386,89 @@ function runGep(epochs::Int, return best, recorder.history end +#feature: has not been tested yet +function runGepExtented(epochs::Int, + population_size::Int, + toolbox::Toolbox, + compute_fitness_::Function; + hof::Int=3, + correction_callback::Union{Function,Nothing}=nothing, + correction_epochs::Int=1, + correction_amount::Real=0.6, + tourni_size::Int=3, + opt_method_const::Symbol=:cg, + optimisation_epochs::Int=500, + selection_mechanism::Function=basic_tournament_selection, + optimizer_function::Union{Function, Nothing}=nothing) + recorder = HistoryRecorder(epochs, Float64) + mating_ = toolbox.gep_probs["mating_size"] + mating_size = Int(ceil(population_size * mating_)) + mating_size = mating_size % 2 == 0 ? mating_size : mating_size - 1 + fits_representation = num_objectives == 1 ? Vector{T}(undef, population_size) : Vector{Tuple}(undef, population_size) + population = generate_population(population_size, toolbox) + next_gen = Vector{eltype(population)}(undef, mating_size) + progBar = Progress(epochs; showspeed=true, desc="Training: ") + prev_best = -1 + for epoch in 1:epochs + perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) + Threads.@threads for i in eachindex(population) + if isnan(population[i].fitness) + population[i].fitness = compute_fitness_(population[i]) + end + end + sort!(population, by=x -> mean(x.fitness)) + try + if (prev_best == -1 || prev_best > population[1].fitness) && epoch % optimisation_epochs == 0 + eqn, result = optimize_constants!(population[1].compiled_function, optimizer_function; + opt_method=opt_method_const, max_iterations=150, n_restarts=5) + population[1].fitness = result + population[1].compiled_function = eqn + prev_best = result + end + catch + @show "Ignored constant opt." + end + Threads.@threads for index in eachindex(population) + fits_representation[index] = population[index].fitness + end + val_loss = compute_fitness_(population[1]; validate=true) + record!(recorder, epoch, fits_representation[1], val_loss, fits_representation) + ProgressMeter.update!(progBar, epoch, showvalues=[ + (:epoch_, @sprintf("%.0f", epoch)), + (:train_loss, @sprintf("%.6e", fits_representation[1])), + (:validation_loss, @sprintf("%.6e", val_loss)) + ]) + if isclose(mean(fits_representation[1]), zero(T)) + break + end + + #ref -> done - dyn select method + if epoch < epochs + selectedMembers = selection_mechanism(fits_representation[1:mating_size], mating_size, tourni_size) + parents = population[selectedMembers.indices] + perform_step!(population, parents, next_gen, toolbox, mating_size) + end + end + best = population[1:hof] + close_recorder!(recorder) - - - - + return best, recorder.history +end end From c768cd2b41682e430ed8e41a08e010251d4b6744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sun, 12 Jan 2025 16:58:30 +1100 Subject: [PATCH 04/41] test flex error --- src/Gep.jl | 258 +++++++++++++------------------------- src/RegressionWrapper.jl | 49 ++++++-- src/Selection.jl | 8 +- src/Util.jl | 2 +- test/Main_min_example.jl | 9 +- test/non_dom_sort_test.jl | 4 +- 6 files changed, 139 insertions(+), 191 deletions(-) diff --git a/src/Gep.jl b/src/Gep.jl index b19609c..790e4a8 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -69,7 +69,7 @@ include("Selection.jl") include("Entities.jl") using .LossFunction -export runGep +export runGep, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy using .GepUtils using .EvoSelection @@ -84,25 +84,61 @@ using DynamicExpressions using Logging using Printf - - const Chromosome = GepEntities.Chromosome const Toolbox = GepEntities.Toolbox -abstract type EvaluationStrategy{R} end +abstract type EvaluationStrategy end -struct StandardRegression{T<:AbstractFloat} <: EvaluationStrategy{T} +struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy operators::OperatorEnum + number_of_objective::Int x_data::AbstractArray{T} y_data::AbstractArray{T} - evaluator::Function + x_data_test::AbstractArray{T} + y_data_test::AbstractArray{T} + loss_function::Function + secOptimizer::Union{Function,Nothing} + break_condition::Union{Function,Nothing} + penalty::T crash_value::T + + function StandardRegressionStrategy{T}(operators::OperatorEnum, + x_data::AbstractArray, + y_data::AbstractArray, + x_data_test::AbstractArray, + y_data_test::AbstractArray, + loss_function::Function; + secOptimizer::Union{Function,Nothing}=nothing, + break_condition::Union{Function,Nothing}=nothing, + penalty::T=zero(T), + crash_value::T=typemax(T)) where T<:AbstractFloat + new(operators, + 1, + x_data, + y_data, + x_data_test, + y_data_test, + loss_function, + secOptimizer, + break_condition, + penalty, + crash_value + ) + end + end -struct GenerEvaluation{R} <: EvaluationStrategy{R} +struct GenericRegressionStrategy <: EvaluationStrategy operators::OperatorEnum - evaluator::Function - crash_value::R + number_of_objective::Int + loss_function::Function + secOptimizer::Union{Function,Nothing} + break_condition::Union{Function,Nothing} + + function GenericRegressionStrategy(operators::OperatorEnum,number_of_objective::Int, loss_function::Function; + secOptimizer::Union{Function,Nothing},break_condition::Union{Function,Nothing}) + new(operators, number_of_objective, loss_function, secOptimizer, break_condition) + end end @@ -131,21 +167,23 @@ Returns the computed fitness value (loss) or crash_value if computation fails - Evaluates the chromosome's compiled function on input data - Returns crash_value if any errors occur during computation """ -@inline function compute_fitness(elem::Chromosome, operators::OperatorEnum, x_data::AbstractArray{T}, - y_data::AbstractArray{T}, loss_function::Function, - crash_value::T; validate::Bool=false) where {T<:AbstractFloat} +@inline function compute_fitness(elem::Chromosome,evalArgs::StandardRegressionStrategy; validate::Bool=false) try if isnan(elem.fitness) || validate - y_pred = elem.compiled_function(x_data, operators) - return loss_function(y_data, y_pred) + y_pred = elem.compiled_function(evalArgs.x_data, evalArgs.operators) + return evalArgs.loss_function(evalArgs.y_data, y_pred) else return elem.fitness end catch e - return crash_value + return evalArgs.crash_value end end +@inline function compute_fitness(elem::Chromosome, evalArgs::GenericRegressionStrategy; validate::Bool=false) + return evalArgs.loss_function(elem, validate) +end + """ perform_step!(population::Vector{Chromosome}, parents::Vector{Chromosome}, next_gen::Vector{Chromosome}, toolbox::Toolbox, mating_size::Int) @@ -231,91 +269,70 @@ Applies correction operations to ensure dimensional homogeneity in chromosomes. end end + """ - runGep(epochs::Int, population_size::Int, operators::OperatorEnum, - x_data::AbstractArray{T}, y_data::AbstractArray{T}, toolbox::Toolbox; - hof::Int=3, x_data_test::Union{AbstractArray{T},Nothing}=nothing, - y_data_test::Union{AbstractArray{T},Nothing}=nothing, - loss_fun_::Union{String,Function}="mae", - correction_callback::Union{Function,Nothing}=nothing, + runGep(epochs::Int, population_size::Int, toolbox::Toolbox, evalStrategy::EvaluationStrategy; + hof::Int=3, correction_callback::Union{Function,Nothing}=nothing, correction_epochs::Int=1, correction_amount::Real=0.6, - tourni_size::Int=3, opt_method_const::Symbol=:cg, - optimisation_epochs::Int=500) where {T<:AbstractFloat} + tourni_size::Int=3) -Main function that executes the GEP algorithm for regression problems. +Main function that executes the GEP algorithm using a specified evaluation strategy. # Arguments - `epochs::Int`: Number of evolutionary epochs to run - `population_size::Int`: Size of the chromosome population -- `operators::OperatorEnum`: Mathematical operators for the DynamicExpressions -- `x_data::AbstractArray{T}`: Training input features -- `y_data::AbstractArray{T}`: Training target values - `toolbox::Toolbox`: Contains genetic operators and algorithm parameters +- `evalStrategy::EvaluationStrategy`: Strategy for evaluating chromosomes, handling fitness computation, and optimization + +# Optional Arguments - `hof::Int=3`: Number of best solutions to return (Hall of Fame size) -- `x_data_test::Union{AbstractArray{T},Nothing}`: Optional test input features -- `y_data_test::Union{AbstractArray{T},Nothing}`: Optional test target values -- `loss_fun_::Union{String,Function}="mae"`: Loss function for fitness computation -- `correction_callback::Union{Function,Nothing}`: Function for dimensional homogeneity correction +- `correction_callback::Union{Function,Nothing}=nothing`: Function for dimensional homogeneity correction - `correction_epochs::Int=1`: Frequency of correction operations - `correction_amount::Real=0.6`: Proportion of population for correction - `tourni_size::Int=3`: Tournament selection size -- `opt_method_const::Symbol=:cg`: Optimization method for constant optimization -- `optimisation_epochs::Int=500`: Frequency of constant optimization # Returns -Tuple{Vector{Chromosome}, Any}: Returns best solutions and training history -Best solutions include both training and test R² scores when test data is provided +`Tuple{Vector{Chromosome}, Any}`: Returns best solutions and training history # Details 1. Initializes population and evolution parameters 2. For each epoch: - - Applies dimensional homogeneity corrections - - Computes fitness for all chromosomes - - Optimizes constants for best solution periodically + - Applies dimensional homogeneity corrections if provided + - Computes fitness for all chromosomes using evaluation strategy + - Sorts population based on fitness + - Applies secondary optimization if specified in strategy - Records training progress - - Performs tournament selection - - Creates new generation through genetic operations -3. Computes final R² scores for best solutions -4. Returns best solutions and training history + - Checks break condition from evaluation strategy + - Performs selection and creates new generation +3. Returns best solutions and training history Progress is monitored through a progress bar showing: - Current epoch - Training loss - Validation loss -Early stopping occurs if perfect R² score is achieved + +The evolution process stops when either: +- Maximum epochs is reached +- Break condition specified in evaluation strategy is met => needs to be informed as break_condition(population, epoch) """ function runGep(epochs::Int, population_size::Int, - operators::OperatorEnum, - x_data::AbstractArray{T}, - y_data::AbstractArray{T}, - toolbox::Toolbox; + toolbox::Toolbox, + evalStrategy::EvaluationStrategy; hof::Int=3, - x_data_test::Union{AbstractArray{T},Nothing}=nothing, - y_data_test::Union{AbstractArray{T},Nothing}=nothing, - loss_fun_::Union{String,Function}="mae", correction_callback::Union{Function,Nothing}=nothing, correction_epochs::Int=1, correction_amount::Real=0.6, - tourni_size::Int=3, - opt_method_const::Symbol=:cg, - optimisation_epochs::Int=500, - selection_mechanism::Function=basic_tournament_selection, - num_objectives::Int=1) where {T<:AbstractFloat} - - loss_fun = typeof(loss_fun_) == String ? get_loss_function(loss_fun_) : loss_fun_ - + tourni_size::Int=3) recorder = HistoryRecorder(epochs, Float64) - function optimizer_function(sub_tree::Node) - y_pred, flag = eval_tree_array(sub_tree, x_data, operators) #another form of the compute fitness - return get_loss_function("mse")(y_pred, y_data) - end mating_ = toolbox.gep_probs["mating_size"] mating_size = Int(ceil(population_size * mating_)) mating_size = mating_size % 2 == 0 ? mating_size : mating_size - 1 - fits_representation = num_objectives == 1 ? Vector{T}(undef, population_size) : Vector{Tuple}(undef, population_size) + fits_representation = evalStrategy.number_of_objective == 1 ? Vector{eltype( + Float64 + )}(undef, population_size) : Vector{Tuple}(undef, population_size) population = generate_population(population_size, toolbox) next_gen = Vector{eltype(population)}(undef, mating_size) @@ -328,119 +345,21 @@ function runGep(epochs::Int, Threads.@threads for i in eachindex(population) if isnan(mean(population[i].fitness)) - population[i].fitness = compute_fitness(population[i], operators, x_data, y_data, loss_fun, typemax(T)) + population[i].fitness = compute_fitness(population[i],evalStrategy) end end sort!(population, by=x -> mean(x.fitness)) - - try - if (prev_best == -1 || prev_best > population[1].fitness) && epoch % optimisation_epochs == 0 - eqn, result = optimize_constants!(population[1].compiled_function, optimizer_function; - opt_method=opt_method_const, max_iterations=150, n_restarts=5) - population[1].fitness = result - population[1].compiled_function = eqn - prev_best = result - end - catch - @show "Ignored constant opt." - end - + Threads.@threads for index in eachindex(population) fits_representation[index] = population[index].fitness end - val_loss = compute_fitness(population[1], operators, x_data_test, y_data_test, loss_fun, typemax(T); validate=true) - record!(recorder, epoch, fits_representation[1], val_loss, fits_representation) - - - ProgressMeter.update!(progBar, epoch, showvalues=[ - (:epoch_, @sprintf("%.0f", epoch)), - (:train_loss, @sprintf("%.6e", fits_representation[1])), - (:validation_loss, @sprintf("%.6e", val_loss)) - ]) - - - if isclose(fits_representation[1], zero(T)) - break - end - - #ref -> done - dyn select method - if epoch < epochs - selectedMembers = selection_mechanism(fits_representation[1:mating_size], mating_size, tourni_size) - parents = population[selectedMembers.indices] - perform_step!(population, parents, next_gen, toolbox, mating_size) - end - - end - - best = population[1:hof] - for elem in best - elem.fitness_r2_train = compute_fitness(elem, operators, x_data, y_data, get_loss_function("r2_score"), zero(T); validate=true) - if !isnothing(x_data_test) - elem.fitness_r2_test = compute_fitness(elem, operators, x_data_test, y_data_test, get_loss_function("r2_score"), zero(T); validate=true) + if !isnothing(evalStrategy.secOptimizer) + evalStrategy.secOptimizer(population,epoch) end - end - close_recorder!(recorder) - return best, recorder.history -end - -#feature: has not been tested yet -function runGepExtented(epochs::Int, - population_size::Int, - toolbox::Toolbox, - compute_fitness_::Function; - hof::Int=3, - correction_callback::Union{Function,Nothing}=nothing, - correction_epochs::Int=1, - correction_amount::Real=0.6, - tourni_size::Int=3, - opt_method_const::Symbol=:cg, - optimisation_epochs::Int=500, - selection_mechanism::Function=basic_tournament_selection, - optimizer_function::Union{Function, Nothing}=nothing) - - recorder = HistoryRecorder(epochs, Float64) - mating_ = toolbox.gep_probs["mating_size"] - mating_size = Int(ceil(population_size * mating_)) - mating_size = mating_size % 2 == 0 ? mating_size : mating_size - 1 - fits_representation = num_objectives == 1 ? Vector{T}(undef, population_size) : Vector{Tuple}(undef, population_size) - - population = generate_population(population_size, toolbox) - next_gen = Vector{eltype(population)}(undef, mating_size) - progBar = Progress(epochs; showspeed=true, desc="Training: ") - - prev_best = -1 - - for epoch in 1:epochs - perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) - - Threads.@threads for i in eachindex(population) - if isnan(population[i].fitness) - population[i].fitness = compute_fitness_(population[i]) - end - end - - sort!(population, by=x -> mean(x.fitness)) - - try - if (prev_best == -1 || prev_best > population[1].fitness) && epoch % optimisation_epochs == 0 - eqn, result = optimize_constants!(population[1].compiled_function, optimizer_function; - opt_method=opt_method_const, max_iterations=150, n_restarts=5) - population[1].fitness = result - population[1].compiled_function = eqn - prev_best = result - end - catch - @show "Ignored constant opt." - end - - Threads.@threads for index in eachindex(population) - fits_representation[index] = population[index].fitness - end - - val_loss = compute_fitness_(population[1]; validate=true) + val_loss = compute_fitness(population[1], evalStrategy; validate=true) record!(recorder, epoch, fits_representation[1], val_loss, fits_representation) @@ -451,13 +370,12 @@ function runGepExtented(epochs::Int, ]) - if isclose(mean(fits_representation[1]), zero(T)) + if !isnothing(evalStrategy.break_condition) && evalStrategy.break_condition(population, epoch) break end - #ref -> done - dyn select method if epoch < epochs - selectedMembers = selection_mechanism(fits_representation[1:mating_size], mating_size, tourni_size) + selectedMembers = selection(fits_representation[1:mating_size], mating_size, tourni_size) parents = population[selectedMembers.indices] perform_step!(population, parents, next_gen, toolbox, mating_size) end @@ -466,9 +384,7 @@ function runGepExtented(epochs::Int, best = population[1:hof] close_recorder!(recorder) - return best, recorder.history end - end diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index 0b99c8c..f7dee49 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -695,29 +695,58 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size, x_train::Ab nothing end - - best, history = runGep(epochs, - population_size, + evalStrat=StandardRegressionStrategy{typeof(first(x_train))}( regressor.operators_, x_train, y_train, - regressor.toolbox_; + !isnothing(x_test) ? x_test : x_train, + !isnothing(y_test) ? y_test : y_train, + get_loss_function(loss_fun) + ) + + best, history = runGep(epochs, + population_size, + regressor.toolbox_, + evalStrat; hof=hof, - x_data_test=!isnothing(x_test) ? x_test : x_train, - y_data_test=!isnothing(y_test) ? y_test : y_train, - loss_fun_=loss_fun, correction_callback=correction_callback, correction_epochs=correction_epochs, correction_amount=correction_amount, - tourni_size=tourni_size, - opt_method_const=opt_method_const, - optimisation_epochs=optimization_epochs) + tourni_size=tourni_size + ) regressor.best_models_ = best regressor.fitness_history_ = history end + +""" +optimizer modes +function optimizer_function(sub_tree::Node) + y_pred, flag = eval_tree_array(sub_tree, x_data, operators) #another form of the compute fitness + return get_loss_function("mse")(y_pred, y_data) +end +""" + + +""" + try + if (prev_best == -1 || prev_best > population[1].fitness) && epoch % optimisation_epochs == 0 + eqn, result = optimize_constants!(population[1].compiled_function, optimizer_function; + opt_method=opt_method_const, max_iterations=150, n_restarts=5) + population[1].fitness = result + population[1].compiled_function = eqn + prev_best = result + end + catch + @show "Ignored constant opt." + end +""" + + + + """ (regressor::GepRegressor)(x_data::AbstractArray; ensemble::Bool=false) diff --git a/src/Selection.jl b/src/Selection.jl index 0300009..59de845 100644 --- a/src/Selection.jl +++ b/src/Selection.jl @@ -1,7 +1,7 @@ module EvoSelection using LinearAlgebra -export selection_NSGA, basic_tournament_selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance +export selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance struct SelectedMembers @@ -10,8 +10,7 @@ struct SelectedMembers end #Note: selection is constructed to allways return a list of indices => {just care about the data not the objects} - -@inline function basic_tournament_selection(population::AbstractArray{T}, number_of_winners::Int, tournament_size::Int) where {T<:Number} +@inline function selection(population::AbstractArray{T}, number_of_winners::Int, tournament_size::Int) where {T<:Number} selected_indices = Vector{Int}(undef, number_of_winners) valid_indices_ = findall(isfinite, population) valid_indices = [] @@ -165,7 +164,8 @@ end return distances end -@inline function selection_NSGA(population::Vector{T}, number_of_winners::Int, tournament_size::Int) where {T<:Tuple} + +@inline function selection(population::Vector{T}) where {T<:Tuple} fronts = calculate_fronts(population) n_fronts = length(fronts) diff --git a/src/Util.jl b/src/Util.jl index d5eeaeb..3c900ed 100644 --- a/src/Util.jl +++ b/src/Util.jl @@ -423,7 +423,7 @@ function compile_djl_datatype(rek_string::Vector, arity_map::OrderedDict, callba return last(stack) end -function retrieve_constants_from_node(node::Node) +@inline function retrieve_constants_from_node(node::Node) constants = AbstractFloat[] for op in node if op isa AbstractNode && op.degree == 0 && op.constant diff --git a/test/Main_min_example.jl b/test/Main_min_example.jl index 3983e14..c5c87e5 100644 --- a/test/Main_min_example.jl +++ b/test/Main_min_example.jl @@ -3,12 +3,13 @@ include("../src/GeneExpressionProgramming.jl") using .GeneExpressionProgramming using Random using Plots +using BenchmarkTools Random.seed!(1) #Define the iterations for the algorithm and the population size -epochs = 1000 -population_size = 1000 +epochs = 100 +population_size = 100000 #Number of features which needs to be inserted number_features = 2 @@ -18,8 +19,9 @@ y_data = @. x_data[:,1] * x_data[:,1] + x_data[:,1] * x_data[:,2] - 2 * x_data[: #define the regressor = GepRegressor(number_features) -fit!(regressor, epochs, population_size, x_data', y_data; loss_fun="mse") +@btime fit!(regressor, epochs, population_size, x_data', y_data; loss_fun="mse") +""" pred = regressor(x_data') @show regressor.best_models_[1].compiled_function @@ -53,3 +55,4 @@ plot!( label="Validation Loss", linewidth=2 ) +""" \ No newline at end of file diff --git a/test/non_dom_sort_test.jl b/test/non_dom_sort_test.jl index a560528..0945987 100644 --- a/test/non_dom_sort_test.jl +++ b/test/non_dom_sort_test.jl @@ -89,7 +89,7 @@ end (1, 2), (2, 1), (1, 1) ]) - selected2 = selection_NSGA(pop2, 10, 0) + selected2 = selection(pop2) @test length(selected2.indices) == 15 @test 15 in selected2.indices # Preserve the best!! alllllways @test length(selected2.fronts) == 5 @@ -100,7 +100,7 @@ end (1, 2, 3), (2, 3, 1), (3, 1, 2), (1, 3, 2), (2, 1, 3), (3, 2, 1) ]) - selected3 = selection_NSGA(pop3, 5, 0) + selected3 = selection(pop3) @test length(selected3.indices) == 9 @test 1 in selected3.indices # The best individual should always be selected @test length(selected3.fronts) == 3 From 3fdb2df901a7bdfa87650038d00f435db3e053f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sun, 12 Jan 2025 21:05:04 +1100 Subject: [PATCH 05/41] test regression test --- src/Gep.jl | 6 ++--- src/RegressionWrapper.jl | 55 ++++++++++++++++++---------------------- test/Main_min_example.jl | 6 ++--- 3 files changed, 30 insertions(+), 37 deletions(-) diff --git a/src/Gep.jl b/src/Gep.jl index 790e4a8..1c816fd 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -338,7 +338,7 @@ function runGep(epochs::Int, next_gen = Vector{eltype(population)}(undef, mating_size) progBar = Progress(epochs; showspeed=true, desc="Training: ") - prev_best = -1 + prev_best = -1.0 for epoch in 1:epochs perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) @@ -356,7 +356,7 @@ function runGep(epochs::Int, end if !isnothing(evalStrategy.secOptimizer) - evalStrategy.secOptimizer(population,epoch) + prev_best=evalStrategy.secOptimizer(population,epoch) end val_loss = compute_fitness(population[1], evalStrategy; validate=true) @@ -375,7 +375,7 @@ function runGep(epochs::Int, end if epoch < epochs - selectedMembers = selection(fits_representation[1:mating_size], mating_size, tourni_size) + selectedMembers = selection(fits_representation, mating_size, tourni_size) parents = population[selectedMembers.indices] perform_step!(population, parents, next_gen, toolbox, mating_size) end diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index f7dee49..981974f 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -164,6 +164,7 @@ using OrderedCollections const Toolbox = GepRegression.GepEntities.Toolbox const TokenDto = SBPUtils.TokenDto +const Chromosome = GepRegression.GepEntities.Chromosome function sqr(x::Vector{T}) where {T<:AbstractFloat} return x .* x @@ -566,7 +567,7 @@ mutable struct GepRegressor toolbox_::Toolbox operators_::OperatorEnum dimension_information_::OrderedDict{Int8,Vector{Float16}} - best_models_::Union{Nothing,Vector{GepRegression.GepEntities.Chromosome}} + best_models_::Union{Nothing,Vector{Chromosome}} fitness_history_::Any token_dto_::Union{TokenDto,Nothing} @@ -674,12 +675,12 @@ Train the GEP regressor model. """ function fit!(regressor::GepRegressor, epochs::Int, population_size, x_train::AbstractArray, y_train::AbstractArray; x_test::Union{AbstractArray,Nothing}=nothing, y_test::Union{AbstractArray,Nothing}=nothing, - optimization_epochs::Int=500, + optimization_epochs::Int=100, hof::Int=3, loss_fun::Union{String,Function}="mse", correction_epochs::Int=1, correction_amount::Real=0.05, tourni_size::Int=3, opt_method_const::Symbol=:cg, target_dimension::Union{Vector{Float16},Nothing}=nothing, - cycles::Int=10 + cycles::Int=10, max_iterations::Int=150, n_starts::Int=5 ) correction_callback = if !isnothing(target_dimension) @@ -695,13 +696,33 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size, x_train::Ab nothing end + @inline function optimizer_function_(sub_tree::Node) + y_pred, flag = eval_tree_array(sub_tree, x_train, regressor.operators_) + return get_loss_function("mse")(y_pred, y_train) + end + + function optimizer_wrapper(population::Vector{Chromosome}, epoch::Int) + try + if epoch % optimization_epochs == 0 + eqn, result = optimize_constants!(population[1].compiled_function, optimizer_function_; + opt_method=opt_method_const, max_iterations=max_iterations, n_restarts=n_starts) + population[1].fitness = result + population[1].compiled_function = eqn + @show "Succesful" + end + catch + @show "Ignored constant opt." + end + end + evalStrat=StandardRegressionStrategy{typeof(first(x_train))}( regressor.operators_, x_train, y_train, !isnothing(x_test) ? x_test : x_train, !isnothing(y_test) ? y_test : y_train, - get_loss_function(loss_fun) + get_loss_function(loss_fun); + secOptimizer=optimizer_wrapper ) best, history = runGep(epochs, @@ -721,32 +742,6 @@ end -""" -optimizer modes -function optimizer_function(sub_tree::Node) - y_pred, flag = eval_tree_array(sub_tree, x_data, operators) #another form of the compute fitness - return get_loss_function("mse")(y_pred, y_data) -end -""" - - -""" - try - if (prev_best == -1 || prev_best > population[1].fitness) && epoch % optimisation_epochs == 0 - eqn, result = optimize_constants!(population[1].compiled_function, optimizer_function; - opt_method=opt_method_const, max_iterations=150, n_restarts=5) - population[1].fitness = result - population[1].compiled_function = eqn - prev_best = result - end - catch - @show "Ignored constant opt." - end -""" - - - - """ (regressor::GepRegressor)(x_data::AbstractArray; ensemble::Bool=false) diff --git a/test/Main_min_example.jl b/test/Main_min_example.jl index c5c87e5..7568ba6 100644 --- a/test/Main_min_example.jl +++ b/test/Main_min_example.jl @@ -3,7 +3,6 @@ include("../src/GeneExpressionProgramming.jl") using .GeneExpressionProgramming using Random using Plots -using BenchmarkTools Random.seed!(1) @@ -19,9 +18,9 @@ y_data = @. x_data[:,1] * x_data[:,1] + x_data[:,1] * x_data[:,2] - 2 * x_data[: #define the regressor = GepRegressor(number_features) -@btime fit!(regressor, epochs, population_size, x_data', y_data; loss_fun="mse") +fit!(regressor, epochs, population_size, x_data', y_data; loss_fun="mse") + -""" pred = regressor(x_data') @show regressor.best_models_[1].compiled_function @@ -55,4 +54,3 @@ plot!( label="Validation Loss", linewidth=2 ) -""" \ No newline at end of file From 380f1bb321a56c7328c6d1f3f3ae0a4b53c00e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Mon, 13 Jan 2025 22:31:18 +1100 Subject: [PATCH 06/41] multi optimization capable --- README.md | 3 +- src/Entities.jl | 11 +-- src/GeneExpressionProgramming.jl | 4 +- src/Gep.jl | 41 +++++------ src/RegressionWrapper.jl | 119 +++++++++++++++++++++++-------- src/Selection.jl | 4 +- src/Util.jl | 41 ++++++++--- test/Main_min_example.jl | 4 +- 8 files changed, 151 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index e32ab1c..6c1cba8 100644 --- a/README.md +++ b/README.md @@ -105,4 +105,5 @@ The repository contains an implementation of the Gene Expression Programming [1] - [ ] Documentation - [x] Naming conventions! - [x] Improve usability for user interaction -- [ ] Next operations: Tail flip, Connection symbol flip, wrapper class for easy usage, config class for predefinition, staggered exploration +- [x] Next operations: Tail flip, Connection symbol flip, staggered exploration +- [ ] Mitigate exception handling in hot paths \ No newline at end of file diff --git a/src/Entities.jl b/src/Entities.jl index f883c5b..bcb6ba5 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -90,8 +90,6 @@ include("Util.jl") using .GepUtils using OrderedCollections - - """ Toolbox @@ -142,7 +140,7 @@ struct Toolbox function Toolbox(gene_count::Int, head_len::Int, symbols::OrderedDict{Int8,Int8}, gene_connections::Vector{Int8}, callbacks::Dict, nodes::OrderedDict, gep_probs::Dict{String,AbstractFloat}; - unary_prob::Real=0.1, fitness_reset::Tuple=(Inf, NaN), preamble_syms=Int8[]) + unary_prob::Real=0.1, fitness_reset::Tuple=((Inf,), (NaN,)), preamble_syms=Int8[]) gene_len = head_len * 2 + 1 headsyms = [key for (key, arity) in symbols if arity == 2] unary_syms = [key for (key, arity) in symbols if arity == 1] @@ -176,12 +174,10 @@ Represents an individual solution in GEP. """ mutable struct Chromosome genes::Vector{Int8} - fitness::Union{AbstractFloat,Tuple} + fitness::Tuple toolbox::Toolbox compiled_function::Any compiled::Bool - fitness_r2_train::AbstractFloat - fitness_r2_test::AbstractFloat expression_raw::Vector{Int8} dimension_homogene::Bool chromo_id::Int @@ -189,11 +185,8 @@ mutable struct Chromosome function Chromosome(genes::Vector{Int8}, toolbox::Toolbox, compile::Bool=false) obj = new() obj.genes = genes - obj.fitness = toolbox.fitness_reset[2] obj.toolbox = toolbox - obj.fitness_r2_train = 0.0 - obj.fitness_r2_test = 0.0 obj.compiled = false obj.dimension_homogene = false obj.chromo_id = -1 diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index 87a6418..aa0add6 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -91,7 +91,7 @@ include("RegressionWrapper.jl") export GepEntities, LossFunction, PhysicalConstants, GepUtils, GepRegression, RegressionWrapper using .GepRegression -export runGep +export runGep, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy using .LossFunction export get_loss_function @@ -104,7 +104,7 @@ export HistoryRecorder, OptimizationHistory, get_history_arrays export train_test_split using .EvoSelection -export selection_NSGA, basic_tournament_selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance +export selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance using .PhysicalConstants diff --git a/src/Gep.jl b/src/Gep.jl index 1c816fd..8afe8dc 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -91,7 +91,7 @@ abstract type EvaluationStrategy end struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy operators::OperatorEnum - number_of_objective::Int + number_of_objectives::Int x_data::AbstractArray{T} y_data::AbstractArray{T} x_data_test::AbstractArray{T} @@ -130,14 +130,14 @@ end struct GenericRegressionStrategy <: EvaluationStrategy operators::OperatorEnum - number_of_objective::Int + number_of_objectives::Int loss_function::Function secOptimizer::Union{Function,Nothing} break_condition::Union{Function,Nothing} - function GenericRegressionStrategy(operators::OperatorEnum,number_of_objective::Int, loss_function::Function; + function GenericRegressionStrategy(operators::OperatorEnum,number_of_objectives::Int, loss_function::Function; secOptimizer::Union{Function,Nothing},break_condition::Union{Function,Nothing}) - new(operators, number_of_objective, loss_function, secOptimizer, break_condition) + new(operators, number_of_objectives, loss_function, secOptimizer, break_condition) end end @@ -169,14 +169,14 @@ Returns the computed fitness value (loss) or crash_value if computation fails """ @inline function compute_fitness(elem::Chromosome,evalArgs::StandardRegressionStrategy; validate::Bool=false) try - if isnan(elem.fitness) || validate + if isnan(mean(elem.fitness)) || validate y_pred = elem.compiled_function(evalArgs.x_data, evalArgs.operators) - return evalArgs.loss_function(evalArgs.y_data, y_pred) + return (evalArgs.loss_function(evalArgs.y_data, y_pred),) else - return elem.fitness + return (elem.fitness,) end catch e - return evalArgs.crash_value + return (evalArgs.crash_value,) end end @@ -323,22 +323,21 @@ function runGep(epochs::Int, correction_callback::Union{Function,Nothing}=nothing, correction_epochs::Int=1, correction_amount::Real=0.6, - tourni_size::Int=3) - recorder = HistoryRecorder(epochs, Float64) - + tourni_size::Int=3, + optimization_epochs::Int=500) + recorder = HistoryRecorder(epochs, Tuple) mating_ = toolbox.gep_probs["mating_size"] mating_size = Int(ceil(population_size * mating_)) mating_size = mating_size % 2 == 0 ? mating_size : mating_size - 1 - fits_representation = evalStrategy.number_of_objective == 1 ? Vector{eltype( - Float64 - )}(undef, population_size) : Vector{Tuple}(undef, population_size) - + fits_representation = Vector{Tuple}(undef, population_size) + + population = generate_population(population_size, toolbox) next_gen = Vector{eltype(population)}(undef, mating_size) progBar = Progress(epochs; showspeed=true, desc="Training: ") - prev_best = -1.0 + prev_best = (typemax(Float64),) for epoch in 1:epochs perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) @@ -355,8 +354,10 @@ function runGep(epochs::Int, fits_representation[index] = population[index].fitness end - if !isnothing(evalStrategy.secOptimizer) - prev_best=evalStrategy.secOptimizer(population,epoch) + if !isnothing(evalStrategy.secOptimizer) && epochs % optimization_epochs == 0 && population[1].fitness correct_genes!( + genes, + start_indices, + expression, + target_dimension, + regressor.token_dto_; + cycles=cycles + ) + else + nothing + end + + + function optimizer_wrapper(population::Vector{Chromosome}) + try + eqn, result = optimize_constants!(population[1].compiled_function, optimizer_function_; + opt_method=opt_method_const, max_iterations=max_iterations, n_restarts=n_starts) + population[1].fitness = result + population[1].compiled_function = eqn + catch + @show "Ignored constant opt." + end + end + + evalStrat = GenericRegressionStrategy( + regressor.operators_, + number_of_objectives, + loss_function; + secOptimizer=optimizer_wrapper, + break_condition=break_condition + ) + + best, history = runGep(epochs, + population_size, + regressor.toolbox_, + evalStrat; + hof=hof, + correction_callback=correction_callback, + correction_epochs=correction_epochs, + correction_amount=correction_amount, + tourni_size=tourni_size, + optimization_epochs=optimization_epochs ) regressor.best_models_ = best @@ -783,10 +844,10 @@ println(sin_info.arity) # 1 """ function list_all_functions() return Dict(sym => ( - function_ = _FUNCTION_LIB_COMMON[sym], - arity = _ARITY_LIB_COMMON[sym], - forward_handler = _FUNCTION_LIB_FORWARD_COMMON[sym], - backward_handler = _FUNCTION_LIB_BACKWARD_COMMON[sym] + function_=_FUNCTION_LIB_COMMON[sym], + arity=_ARITY_LIB_COMMON[sym], + forward_handler=_FUNCTION_LIB_FORWARD_COMMON[sym], + backward_handler=_FUNCTION_LIB_BACKWARD_COMMON[sym] ) for sym in keys(_FUNCTION_LIB_COMMON)) end @@ -984,13 +1045,13 @@ update_function!(:custom_func, # Throws - ArgumentError if the function symbol doesn't exist or parameters are invalid """ -function update_function!(sym::Symbol; - func::Union{Function,Nothing}=nothing, - arity::Union{Int8,Nothing}=nothing, - forward_handler::Union{Function,Nothing}=nothing, - backward_handler::Union{Function,Nothing}=nothing) +function update_function!(sym::Symbol; + func::Union{Function,Nothing}=nothing, + arity::Union{Int8,Nothing}=nothing, + forward_handler::Union{Function,Nothing}=nothing, + backward_handler::Union{Function,Nothing}=nothing) haskey(_FUNCTION_LIB_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) - + if !isnothing(func) set_function!(sym, func) end @@ -1003,7 +1064,7 @@ function update_function!(sym::Symbol; if !isnothing(backward_handler) set_backward_handler!(sym, backward_handler) end - + return nothing end diff --git a/src/Selection.jl b/src/Selection.jl index 59de845..2f7b7b0 100644 --- a/src/Selection.jl +++ b/src/Selection.jl @@ -10,9 +10,9 @@ struct SelectedMembers end #Note: selection is constructed to allways return a list of indices => {just care about the data not the objects} -@inline function selection(population::AbstractArray{T}, number_of_winners::Int, tournament_size::Int) where {T<:Number} +@inline function selection(population::AbstractArray{Tuple}, number_of_winners::Int, tournament_size::Int) selected_indices = Vector{Int}(undef, number_of_winners) - valid_indices_ = findall(isfinite, population) + valid_indices_ = findall(x -> isfinite(x[1]), population) valid_indices = [] doubles = Set() for elem in valid_indices_ diff --git a/src/Util.jl b/src/Util.jl index 3c900ed..dc73d19 100644 --- a/src/Util.jl +++ b/src/Util.jl @@ -108,13 +108,13 @@ using Random using Base.Threads: @spawn -struct OptimizationHistory{T<:AbstractFloat} +struct OptimizationHistory{T<:Union{AbstractFloat,Tuple}} train_loss::Vector{T} val_loss::Vector{T} train_mean::Vector{T} train_std::Vector{T} - function OptimizationHistory(epochs::Int, ::Type{T}) where {T<:AbstractFloat} + function OptimizationHistory(epochs::Int, ::Type{T}) where {T<:Union{AbstractFloat,Tuple}} return new{T}( Vector{T}(undef, epochs), Vector{T}(undef, epochs), @@ -242,12 +242,12 @@ final_history = recorder.history - Supports any AbstractFloat type - Channel depth can be adjusted for different recording patterns """ -struct HistoryRecorder{T<:AbstractFloat} +struct HistoryRecorder{T<:Union{AbstractFloat,Tuple}} channel::Channel{Tuple{Int,T,T,Vector{T}}} task::Task history::OptimizationHistory{T} - function HistoryRecorder(epochs::Int, ::Type{T}; buffer_size::Int=32) where {T<:AbstractFloat} + function HistoryRecorder(epochs::Int, ::Type{T}; buffer_size::Int=32) where {T<:Union{AbstractFloat,Tuple}} history = OptimizationHistory(epochs, T) channel = Channel{Tuple{Int,T,T,Vector{T}}}(buffer_size) task = @spawn record_history!(channel, history) @@ -255,17 +255,37 @@ struct HistoryRecorder{T<:AbstractFloat} end end + +@inline function tuple_agg(entries::Vector{T}, fun::Function) where {T<:Tuple} + isempty(entries) && return entries[1] + N = length(first(entries)) + L = length(entries) + + vectors = tuple(i -> Vector{Float64}(undef, L), N) + + for (j, entry) in enumerate(entries) + try + for i in 1:length(entry) + vectors[i][j] = entry[i] + end + catch + + end + end + return tuple(i -> fun(vectors[i]), N) +end + # Usage in record_history! @inline function record_history!( channel::Channel{Tuple{Int,T,T,Vector{T}}}, history::OptimizationHistory{T} -) where {T<:AbstractFloat} +) where {T<:Union{AbstractFloat,Tuple}} for (epoch, train_loss, val_loss, fit_vector) in channel @inbounds begin - history.train_loss[epoch] = train_loss - history.val_loss[epoch] = val_loss - history.train_mean[epoch] = mean(fit_vector) - history.train_std[epoch] = std(fit_vector) + history.train_loss[epoch] = train_loss + history.val_loss[epoch] = val_loss + #history.train_mean[epoch] = tuple_agg(fit_vector,mean) + #history.train_std[epoch] = tuple_agg(fit_vector, std) end end end @@ -276,11 +296,10 @@ end train_loss::T, val_loss::T, fit_vector::Vector{T} -) where {T<:AbstractFloat} +) where {T<:Union{AbstractFloat,Tuple}} put!(recorder.channel, (epoch, train_loss, val_loss, fit_vector)) end - @inline function close_recorder!(recorder::HistoryRecorder) close(recorder.channel) wait(recorder.task) diff --git a/test/Main_min_example.jl b/test/Main_min_example.jl index 7568ba6..9aa2de5 100644 --- a/test/Main_min_example.jl +++ b/test/Main_min_example.jl @@ -7,8 +7,8 @@ using Plots Random.seed!(1) #Define the iterations for the algorithm and the population size -epochs = 100 -population_size = 100000 +epochs = 1000 +population_size = 1000 #Number of features which needs to be inserted number_features = 2 From bfd459afee99302a9463aefec2440835684ab40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Tue, 14 Jan 2025 08:31:07 +1100 Subject: [PATCH 07/41] multi optimization capable --- src/Entities.jl | 7 ++++++- src/RegressionWrapper.jl | 8 ++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index bcb6ba5..414c70d 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -140,7 +140,12 @@ struct Toolbox function Toolbox(gene_count::Int, head_len::Int, symbols::OrderedDict{Int8,Int8}, gene_connections::Vector{Int8}, callbacks::Dict, nodes::OrderedDict, gep_probs::Dict{String,AbstractFloat}; - unary_prob::Real=0.1, fitness_reset::Tuple=((Inf,), (NaN,)), preamble_syms=Int8[]) + unary_prob::Real=0.1, preamble_syms=Int8[], number_of_objectives::Int=1) + + fitness_reset= ( + ntuple(_ -> Inf, number_of_objectives), + ntuple(_ -> NaN, number_of_objectives) + ) gene_len = head_len * 2 + 1 headsyms = [key for (key, arity) in symbols if arity == 2] unary_syms = [key for (key, arity) in symbols if arity == 1] diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index f2311c9..3dd51b7 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -583,7 +583,8 @@ mutable struct GepRegressor gene_count::Int=2, head_len::Int=10, preamble_syms::Vector{Symbol}=Symbol[], - max_permutations_lib::Int=10000, rounds::Int=4 + max_permutations_lib::Int=10000, rounds::Int=4, + number_of_objectives::Int=1 ) entered_features_ = isempty(entered_features) ? @@ -636,7 +637,7 @@ mutable struct GepRegressor end toolbox = GepRegression.GepEntities.Toolbox(gene_count, head_len, utilized_symbols, gene_connections_, - callbacks, nodes, GENE_COMMON_PROBS; preamble_syms=preamble_syms_) + callbacks, nodes, GENE_COMMON_PROBS; preamble_syms=preamble_syms, number_of_objectives=number_of_objectives) obj = new() obj.toolbox_ = toolbox @@ -739,7 +740,6 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, x_trai end function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, loss_function::Function; - number_of_objectives::Int=1, optimizer_function_::Union{Function,Nothing}=nothing, optimization_epochs::Int=100, hof::Int=3, @@ -779,7 +779,7 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, loss_f evalStrat = GenericRegressionStrategy( regressor.operators_, - number_of_objectives, + length(regressor.toolbox_.fitness_reset[1]), loss_function; secOptimizer=optimizer_wrapper, break_condition=break_condition From 670d44e678f8eddcf2f44b2eb548063f5b394028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Tue, 14 Jan 2025 10:53:06 +1100 Subject: [PATCH 08/41] add example multiobjective --- src/Entities.jl | 2 +- src/Gep.jl | 53 +++++++++++++++++++-------------- src/RegressionWrapper.jl | 4 +-- src/Util.jl | 7 +---- test/Main_min_bench.jl | 58 +++++++++++++++++++++++++++++++++++++ test/Main_min_example_mo.jl | 46 +++++++++++++++++++++++++++++ 6 files changed, 139 insertions(+), 31 deletions(-) create mode 100644 test/Main_min_bench.jl create mode 100644 test/Main_min_example_mo.jl diff --git a/src/Entities.jl b/src/Entities.jl index 414c70d..243c29f 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -236,6 +236,7 @@ function compile_expression!(chromosome::Chromosome; force_compile::Bool=false) chromosome.compiled = true catch chromosome.fitness = chromosome.toolbox.fitness_reset[1] + chromosome.expression_raw = expression end end end @@ -333,7 +334,6 @@ function _karva_raw(chromosome::Chromosome) return vcat(rolled_indices...) end - """ generate_gene(headsyms::Vector{Int8}, tailsyms::Vector{Int8}, headlen::Int; unarys::Vector{Int8}=[], unary_prob::Real=0.2) diff --git a/src/Gep.jl b/src/Gep.jl index 8afe8dc..4b0ad83 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -83,6 +83,7 @@ using OrderedCollections using DynamicExpressions using Logging using Printf +using Base.Threads: SpinLock const Chromosome = GepEntities.Chromosome const Toolbox = GepEntities.Toolbox @@ -111,19 +112,19 @@ struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy secOptimizer::Union{Function,Nothing}=nothing, break_condition::Union{Function,Nothing}=nothing, penalty::T=zero(T), - crash_value::T=typemax(T)) where T<:AbstractFloat - new(operators, + crash_value::T=typemax(T)) where {T<:AbstractFloat} + new(operators, 1, - x_data, - y_data, - x_data_test, + x_data, + y_data, + x_data_test, y_data_test, loss_function, secOptimizer, break_condition, penalty, crash_value - ) + ) end end @@ -135,8 +136,8 @@ struct GenericRegressionStrategy <: EvaluationStrategy secOptimizer::Union{Function,Nothing} break_condition::Union{Function,Nothing} - function GenericRegressionStrategy(operators::OperatorEnum,number_of_objectives::Int, loss_function::Function; - secOptimizer::Union{Function,Nothing},break_condition::Union{Function,Nothing}) + function GenericRegressionStrategy(operators::OperatorEnum, number_of_objectives::Int, loss_function::Function; + secOptimizer::Union{Function,Nothing}, break_condition::Union{Function,Nothing}) new(operators, number_of_objectives, loss_function, secOptimizer, break_condition) end end @@ -167,7 +168,7 @@ Returns the computed fitness value (loss) or crash_value if computation fails - Evaluates the chromosome's compiled function on input data - Returns crash_value if any errors occur during computation """ -@inline function compute_fitness(elem::Chromosome,evalArgs::StandardRegressionStrategy; validate::Bool=false) +@inline function compute_fitness(elem::Chromosome, evalArgs::StandardRegressionStrategy; validate::Bool=false) try if isnan(mean(elem.fitness)) || validate y_pred = elem.compiled_function(evalArgs.x_data, evalArgs.operators) @@ -180,7 +181,7 @@ Returns the computed fitness value (loss) or crash_value if computation fails end end -@inline function compute_fitness(elem::Chromosome, evalArgs::GenericRegressionStrategy; validate::Bool=false) +@inline function compute_fitness(elem::Chromosome, evalArgs::GenericRegressionStrategy; validate::Bool=false) return evalArgs.loss_function(elem, validate) end @@ -254,13 +255,13 @@ Applies correction operations to ensure dimensional homogeneity in chromosomes. if !isnothing(correction_callback) && epoch % correction_epochs == 0 pop_amount = Int(ceil(length(population) * correction_amount)) - Threads.@threads for i in 1:pop_amount + Threads.@threads for i in 1:pop_amount if !(population[i].dimension_homogene) && population[i].compiled distance, correction = correction_callback(population[i].genes, population[i].toolbox.gen_start_indices, population[i].expression_raw) if correction compile_expression!(population[i]; force_compile=true) - population[i].dimension_homogene = true + population[i].dimension_homogene = true else population[i].fitness += distance end @@ -331,30 +332,38 @@ function runGep(epochs::Int, mating_size = Int(ceil(population_size * mating_)) mating_size = mating_size % 2 == 0 ? mating_size : mating_size - 1 fits_representation = Vector{Tuple}(undef, population_size) - - + fit_cache = Dict{Vector{Int8},Tuple}() + cache_lock = SpinLock() + population = generate_population(population_size, toolbox) next_gen = Vector{eltype(population)}(undef, mating_size) progBar = Progress(epochs; showspeed=true, desc="Training: ") prev_best = (typemax(Float64),) - for epoch in 1:epochs perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) Threads.@threads for i in eachindex(population) - if isnan(mean(population[i].fitness)) - population[i].fitness = compute_fitness(population[i],evalStrategy) + cache_value = get(fit_cache, population[i].expression_raw, nothing) + if isnan(mean(population[i].fitness)) + if isnothing(cache_value) + population[i].fitness = compute_fitness(population[i], evalStrategy) + lock(cache_lock) + fit_cache[population[i].expression_raw] = population[i].fitness + unlock(cache_lock) + else + population[i].fitness = cache_value + same+=1 + end end end - sort!(population, by=x -> mean(x.fitness)) - + Threads.@threads for index in eachindex(population) - fits_representation[index] = population[index].fitness + fits_representation[index] = population[index].fitness end - if !isnothing(evalStrategy.secOptimizer) && epochs % optimization_epochs == 0 && population[1].fitness Vector{Float64}(undef, L), N) + vectors = ntuple(i -> Vector{Float64}(undef, L), N) for (j, entry) in enumerate(entries) - try for i in 1:length(entry) vectors[i][j] = entry[i] end - catch - - end end return tuple(i -> fun(vectors[i]), N) end -# Usage in record_history! @inline function record_history!( channel::Channel{Tuple{Int,T,T,Vector{T}}}, history::OptimizationHistory{T} diff --git a/test/Main_min_bench.jl b/test/Main_min_bench.jl new file mode 100644 index 0000000..c5c87e5 --- /dev/null +++ b/test/Main_min_bench.jl @@ -0,0 +1,58 @@ +include("../src/GeneExpressionProgramming.jl") + +using .GeneExpressionProgramming +using Random +using Plots +using BenchmarkTools + +Random.seed!(1) + +#Define the iterations for the algorithm and the population size +epochs = 100 +population_size = 100000 + +#Number of features which needs to be inserted +number_features = 2 + +x_data = randn(Float64, 100, number_features) +y_data = @. x_data[:,1] * x_data[:,1] + x_data[:,1] * x_data[:,2] - 2 * x_data[:,2] * x_data[:,2] + +#define the +regressor = GepRegressor(number_features) +@btime fit!(regressor, epochs, population_size, x_data', y_data; loss_fun="mse") + +""" +pred = regressor(x_data') + +@show regressor.best_models_[1].compiled_function +@show regressor.best_models_[1].fitness + + +#Making a nice plot - data vs +pred_vs_actual = scatter(vec(pred), vec(y_data), +xlabel="Actual Values", +ylabel="Predicted Values", +label="Predictions ", +title="Predictions vs Actual - Symbolic Regression"); + + +plot!(pred_vs_actual, vec(y_data), vec(y_data), +label="Prediction Comparison", +color=:red) + +#train loss vs validation loss +train_validation = plot( + regressor.fitness_history_.train_loss, + label="Training Loss", + ylabel="Loss", + xlabel="Epoch", + linewidth=2 +); + +plot!( + train_validation, + regressor.fitness_history_.val_loss, + label="Validation Loss", + linewidth=2 +) +""" \ No newline at end of file diff --git a/test/Main_min_example_mo.jl b/test/Main_min_example_mo.jl new file mode 100644 index 0000000..960e211 --- /dev/null +++ b/test/Main_min_example_mo.jl @@ -0,0 +1,46 @@ +include("../src/GeneExpressionProgramming.jl") + +using .GeneExpressionProgramming +using Random +using Plots +using DynamicExpressions +using Statistics + +const Chromosome = GepEntities.Chromosome + + +Random.seed!(1) + +#Define the iterations for the algorithm and the population size +epochs = 1000 +population_size = 1000 + +#Number of features which needs to be inserted +number_features = 2 + +x_data = randn(Float64, 100, number_features) +y_data = @. x_data[:,1] * x_data[:,1] + x_data[:,1] * x_data[:,2] - 2 * x_data[:,2] * x_data[:,2] + +#define the +regressor = GepRegressor(number_features; number_of_objectives=2) + +@inline function loss_new(elem,validate::Bool) + try + if isnan(mean(elem.fitness)) || validate + y_pred = elem.compiled_function(x_data', regressor.operators_) + return (get_loss_function("mse")(y_data, y_pred), length(elem.expression_raw)*0.01) + else + return (elem.fitness,length(elem.expression_raw)*elem.fitness) + end + catch e + return (typemax(Float64),typemax(Float64)) + end +end + +fit!(regressor, epochs, population_size, loss_new) + + +pred = regressor(x_data') + +@show regressor.best_models_[1].compiled_function +@show regressor.best_models_[1].fitness \ No newline at end of file From 5eff99be5507c4f9ed8c40330672d238972259a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Tue, 14 Jan 2025 10:55:53 +1100 Subject: [PATCH 09/41] add example multiobjective --- src/Gep.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Gep.jl b/src/Gep.jl index 4b0ad83..d35b621 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -344,8 +344,8 @@ function runGep(epochs::Int, perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) Threads.@threads for i in eachindex(population) - cache_value = get(fit_cache, population[i].expression_raw, nothing) if isnan(mean(population[i].fitness)) + cache_value = get(fit_cache, population[i].expression_raw, nothing) if isnothing(cache_value) population[i].fitness = compute_fitness(population[i], evalStrategy) lock(cache_lock) @@ -353,7 +353,6 @@ function runGep(epochs::Int, unlock(cache_lock) else population[i].fitness = cache_value - same+=1 end end end From 3f1d78d7e56079b69e8f34b5e8fc8327d54f948d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Tue, 14 Jan 2025 11:29:14 +1100 Subject: [PATCH 10/41] handle dubplicates properly --- src/Entities.jl | 2 +- src/Gep.jl | 6 ++++-- test/Main_min_bench.jl | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index 243c29f..236976e 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -195,6 +195,7 @@ mutable struct Chromosome obj.compiled = false obj.dimension_homogene = false obj.chromo_id = -1 + obj.expression_raw = Int8[] if compile compile_expression!(obj) end @@ -236,7 +237,6 @@ function compile_expression!(chromosome::Chromosome; force_compile::Bool=false) chromosome.compiled = true catch chromosome.fitness = chromosome.toolbox.fitness_reset[1] - chromosome.expression_raw = expression end end end diff --git a/src/Gep.jl b/src/Gep.jl index d35b621..6a57c42 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -84,6 +84,7 @@ using DynamicExpressions using Logging using Printf using Base.Threads: SpinLock +using .Threads const Chromosome = GepEntities.Chromosome const Toolbox = GepEntities.Toolbox @@ -338,9 +339,9 @@ function runGep(epochs::Int, population = generate_population(population_size, toolbox) next_gen = Vector{eltype(population)}(undef, mating_size) progBar = Progress(epochs; showspeed=true, desc="Training: ") - prev_best = (typemax(Float64),) for epoch in 1:epochs + same = Atomic{Int}(0) perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) Threads.@threads for i in eachindex(population) @@ -352,12 +353,12 @@ function runGep(epochs::Int, fit_cache[population[i].expression_raw] = population[i].fitness unlock(cache_lock) else + atomic_add!(same, 1) population[i].fitness = cache_value end end end sort!(population, by=x -> mean(x.fitness)) - Threads.@threads for index in eachindex(population) fits_representation[index] = population[index].fitness end @@ -374,6 +375,7 @@ function runGep(epochs::Int, ProgressMeter.update!(progBar, epoch, showvalues=[ (:epoch_, @sprintf("%.0f", epoch)), + (:duplicates_per_epoch, @sprintf("%.0f", same[])), (:train_loss, @sprintf("%.6e", mean(fits_representation[1]))), (:validation_loss, @sprintf("%.6e", mean(val_loss))) ]) diff --git a/test/Main_min_bench.jl b/test/Main_min_bench.jl index c5c87e5..3c84545 100644 --- a/test/Main_min_bench.jl +++ b/test/Main_min_bench.jl @@ -8,13 +8,13 @@ using BenchmarkTools Random.seed!(1) #Define the iterations for the algorithm and the population size -epochs = 100 -population_size = 100000 +epochs = 1000 +population_size = 1500 #Number of features which needs to be inserted number_features = 2 -x_data = randn(Float64, 100, number_features) +x_data = randn(Float64, 10000, number_features) y_data = @. x_data[:,1] * x_data[:,1] + x_data[:,1] * x_data[:,2] - 2 * x_data[:,2] * x_data[:,2] #define the From 3637f53179ba8ca506b065c2113e6404d95deff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Tue, 14 Jan 2025 11:55:19 +1100 Subject: [PATCH 11/41] adapt test to new interface --- paper/ConstraintViaSBP.jl | 28 +++++++++++++++++++++------- src/Gep.jl | 2 +- test/Main_min_example_mo.jl | 2 +- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/paper/ConstraintViaSBP.jl b/paper/ConstraintViaSBP.jl index 2160629..312d869 100644 --- a/paper/ConstraintViaSBP.jl +++ b/paper/ConstraintViaSBP.jl @@ -9,6 +9,17 @@ using Random using Logging using Dates using JSON +using Statistics + + +function loss_new(eqn::Node, operators::OperatorEnum, x_data::AbstractArray, y_data::AbstractArray) + try + y_pred = eqn(x_data, operators) + return get_loss_function("r2_score")(y_data, y_pred) + catch e + return zero(Float64) + end +end function setup_logger(log_file_path::String) @@ -73,7 +84,7 @@ function main() if case_name in keys(case_data) @show ("Current case: ", case_name) #gep_params - epochs = 1000 + epochs = 100 population_size = 1500 results = DataFrame(Seed=[], @@ -88,7 +99,7 @@ function main() println(feature_names) println(case_name) phy_dims = get_feature_dims_json(case_data, feature_names, case_name) - phy_dims = Dict{Symbol, Vector{Float16}}( Symbol(x_n) => dim_n for (x_n, dim_n) in phy_dims) + phy_dims = Dict{Symbol,Vector{Float16}}(Symbol(x_n) => dim_n for (x_n, dim_n) in phy_dims) target_dim = get_target_dim_json(case_data, case_name) print(phy_dims) @@ -104,23 +115,26 @@ function main() start_time = time_ns() - regressor = GepRegressor(num_cols-1; + regressor = GepRegressor(num_cols - 1; considered_dimensions=phy_dims, entered_non_terminals=[:+, :-, :*, :/, :sqrt, :sin, :cos, :exp, :log], max_permutations_lib=10000, rounds=7) #perform the regression by entering epochs, population_size, the feature cols, the target col and the loss function fit!(regressor, epochs, population_size, x_train', y_train; - x_test=x_test', y_test=y_test', + x_test=x_test', y_test=y_test, loss_fun="mse", target_dimension=target_dim) end_time = (time_ns() - start_time) / 1e9 elem = regressor.best_models_[1] + fitness_r2_train = loss_new(elem.compiled_function, regressor.operators_, x_train', y_train) + fitness_r2_test = loss_new(elem.compiled_function, regressor.operators_, x_test', y_test) + #log_results - push!(results, (seed, case_name, noise_level, elem.fitness, string(elem.compiled_function), - elem.fitness_r2_train, elem.fitness_r2_test, end_time, elem.dimension_homogene, target_dim)) + push!(results, (seed, case_name, noise_level, mean(elem.fitness), string(elem.compiled_function), + fitness_r2_train, fitness_r2_test, end_time, elem.dimension_homogene, target_dim)) - @show elem.fitness_r2_test + @show fitness_r2_test save_results_to_csv(file_name_save, results) end end diff --git a/src/Gep.jl b/src/Gep.jl index 6a57c42..04b34c4 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -264,7 +264,7 @@ Applies correction operations to ensure dimensional homogeneity in chromosomes. compile_expression!(population[i]; force_compile=true) population[i].dimension_homogene = true else - population[i].fitness += distance + #population[i].fitness += distance end end end diff --git a/test/Main_min_example_mo.jl b/test/Main_min_example_mo.jl index 960e211..bf53fe7 100644 --- a/test/Main_min_example_mo.jl +++ b/test/Main_min_example_mo.jl @@ -13,7 +13,7 @@ Random.seed!(1) #Define the iterations for the algorithm and the population size epochs = 1000 -population_size = 1000 +population_size = 10000 #Number of features which needs to be inserted number_features = 2 From 5f99a97fe8d6caca4e6694a8bd8457f31e5af6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Tue, 14 Jan 2025 11:57:42 +1100 Subject: [PATCH 12/41] adapt test to new interface --- paper/ConstraintViaSBP.jl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/paper/ConstraintViaSBP.jl b/paper/ConstraintViaSBP.jl index 312d869..b7d1bb2 100644 --- a/paper/ConstraintViaSBP.jl +++ b/paper/ConstraintViaSBP.jl @@ -118,12 +118,23 @@ function main() regressor = GepRegressor(num_cols - 1; considered_dimensions=phy_dims, entered_non_terminals=[:+, :-, :*, :/, :sqrt, :sin, :cos, :exp, :log], - max_permutations_lib=10000, rounds=7) + max_permutations_lib=10000, rounds=7,number_of_objectives=2) + + @inline function loss_new_(elem, validate::Bool) + try + if isnan(mean(elem.fitness)) || validate + y_pred = elem.compiled_function(x_train', regressor.operators_) + return (get_loss_function("mse")(y_train, y_pred), length(elem.expression_raw) * 0.01) + else + return (elem.fitness, length(elem.expression_raw) * elem.fitness) + end + catch e + return (typemax(Float64), typemax(Float64)) + end + end #perform the regression by entering epochs, population_size, the feature cols, the target col and the loss function - fit!(regressor, epochs, population_size, x_train', y_train; - x_test=x_test', y_test=y_test, - loss_fun="mse", target_dimension=target_dim) + fit!(regressor, epochs, population_size, loss_new_; target_dimension=target_dim) end_time = (time_ns() - start_time) / 1e9 elem = regressor.best_models_[1] From f5966eccbbc43e0beaeb5248d43038e9d2d9b34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Tue, 14 Jan 2025 13:53:50 +1100 Subject: [PATCH 13/41] change files --- paper/ConstraintViaSBP.jl | 4 +++- src/Entities.jl | 2 +- src/GeneExpressionProgramming.jl | 2 +- src/Gep.jl | 7 +++++- src/Selection.jl | 7 +++--- src/Util.jl | 2 +- test/Entity_test.jl | 16 +------------- test/Main_min_bench.jl | 38 +------------------------------- test/Main_min_example.jl | 8 +++---- test/Main_min_example_mo.jl | 2 +- test/Main_min_with_csv.jl | 4 ++-- test/non_dom_sort_test.jl | 4 ++-- 12 files changed, 27 insertions(+), 69 deletions(-) diff --git a/paper/ConstraintViaSBP.jl b/paper/ConstraintViaSBP.jl index b7d1bb2..1434583 100644 --- a/paper/ConstraintViaSBP.jl +++ b/paper/ConstraintViaSBP.jl @@ -120,6 +120,7 @@ function main() entered_non_terminals=[:+, :-, :*, :/, :sqrt, :sin, :cos, :exp, :log], max_permutations_lib=10000, rounds=7,number_of_objectives=2) + """next test scenario @inline function loss_new_(elem, validate::Bool) try if isnan(mean(elem.fitness)) || validate @@ -132,9 +133,10 @@ function main() return (typemax(Float64), typemax(Float64)) end end + """ #perform the regression by entering epochs, population_size, the feature cols, the target col and the loss function - fit!(regressor, epochs, population_size, loss_new_; target_dimension=target_dim) + fit!(regressor, epochs, population_size; target_dimension=target_dim) end_time = (time_ns() - start_time) / 1e9 elem = regressor.best_models_[1] diff --git a/src/Entities.jl b/src/Entities.jl index 236976e..1f485c9 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -263,7 +263,7 @@ Set chromosome's fitness value. - `chromosome::Chromosome`: Target chromosome - `value::AbstractFloat`: New fitness value """ -function set_fitness!(chromosome::Chromosome, value::AbstractFloat) +function set_fitness!(chromosome::Chromosome, value::Tuple) chromosome.fitness = value end diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index aa0add6..f06928d 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -104,7 +104,7 @@ export HistoryRecorder, OptimizationHistory, get_history_arrays export train_test_split using .EvoSelection -export selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance +export tournament_selection, nsga_selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance using .PhysicalConstants diff --git a/src/Gep.jl b/src/Gep.jl index 04b34c4..bcd2dc4 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -386,7 +386,12 @@ function runGep(epochs::Int, end if epoch < epochs - selectedMembers = selection(fits_representation, mating_size, tourni_size) + if length(fits_representation[1]) == 1 + selectedMembers = tournament_selection(fits_representation, mating_size, tourni_size) + else + selectedMembers = nsga_selection(fits_representation) + end + parents = population[selectedMembers.indices] perform_step!(population, parents, next_gen, toolbox, mating_size) end diff --git a/src/Selection.jl b/src/Selection.jl index 2f7b7b0..2d36850 100644 --- a/src/Selection.jl +++ b/src/Selection.jl @@ -1,7 +1,7 @@ module EvoSelection using LinearAlgebra -export selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance +export tournament_selection, nsga_selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance struct SelectedMembers @@ -10,7 +10,7 @@ struct SelectedMembers end #Note: selection is constructed to allways return a list of indices => {just care about the data not the objects} -@inline function selection(population::AbstractArray{Tuple}, number_of_winners::Int, tournament_size::Int) +@inline function tournament_selection(population::AbstractArray{Tuple}, number_of_winners::Int, tournament_size::Int) selected_indices = Vector{Int}(undef, number_of_winners) valid_indices_ = findall(x -> isfinite(x[1]), population) valid_indices = [] @@ -165,7 +165,7 @@ end end -@inline function selection(population::Vector{T}) where {T<:Tuple} +@inline function nsga_selection(population::Vector{T}) where {T<:Tuple} fronts = calculate_fronts(population) n_fronts = length(fronts) @@ -194,4 +194,5 @@ end end + end diff --git a/src/Util.jl b/src/Util.jl index a6866ee..b265ebb 100644 --- a/src/Util.jl +++ b/src/Util.jl @@ -542,7 +542,7 @@ See also: [`DynamicExpressions.Node`](@ref), [`Optim.optimize`](@ref), [`LineSea n_restarts::Int=3 ) - nconst = count_constants(node) + nconst = count_constant_nodes(node) if nconst == 0 return node, 0.0 diff --git a/test/Entity_test.jl b/test/Entity_test.jl index 7cafb33..75ec447 100644 --- a/test/Entity_test.jl +++ b/test/Entity_test.jl @@ -67,7 +67,7 @@ end toolbox.gene_count * (2 * toolbox.head_len + 1)) @test chromosome.compiled == true @test chromosome.dimension_homogene == false - @test isnan(chromosome.fitness) + @test isnan(chromosome.fitness[1]) end @testset "Function Compilation" begin @@ -142,18 +142,4 @@ end @test all(x -> x isa Chromosome, population) @test length(unique([p.genes for p in population])) == population_size end - - @testset "History Recording" begin - toolbox = create_test_toolbox() - population_size = 10 - population = generate_population(population_size, toolbox) - - for (i, chromo) in enumerate(population) - set_fitness!(chromo, Float64(i)) - @test fitness(chromo) == Float64(i) - end - - sorted_pop = sort(population, by=fitness) - @test issorted([fitness(c) for c in sorted_pop]) - end end \ No newline at end of file diff --git a/test/Main_min_bench.jl b/test/Main_min_bench.jl index 3c84545..3110154 100644 --- a/test/Main_min_bench.jl +++ b/test/Main_min_bench.jl @@ -19,40 +19,4 @@ y_data = @. x_data[:,1] * x_data[:,1] + x_data[:,1] * x_data[:,2] - 2 * x_data[: #define the regressor = GepRegressor(number_features) -@btime fit!(regressor, epochs, population_size, x_data', y_data; loss_fun="mse") - -""" -pred = regressor(x_data') - -@show regressor.best_models_[1].compiled_function -@show regressor.best_models_[1].fitness - - -#Making a nice plot - data vs -pred_vs_actual = scatter(vec(pred), vec(y_data), -xlabel="Actual Values", -ylabel="Predicted Values", -label="Predictions ", -title="Predictions vs Actual - Symbolic Regression"); - - -plot!(pred_vs_actual, vec(y_data), vec(y_data), -label="Prediction Comparison", -color=:red) - -#train loss vs validation loss -train_validation = plot( - regressor.fitness_history_.train_loss, - label="Training Loss", - ylabel="Loss", - xlabel="Epoch", - linewidth=2 -); - -plot!( - train_validation, - regressor.fitness_history_.val_loss, - label="Validation Loss", - linewidth=2 -) -""" \ No newline at end of file +@btime fit!(regressor, epochs, population_size, x_data', y_data; loss_fun="mse") \ No newline at end of file diff --git a/test/Main_min_example.jl b/test/Main_min_example.jl index 9aa2de5..a1d89f2 100644 --- a/test/Main_min_example.jl +++ b/test/Main_min_example.jl @@ -7,8 +7,8 @@ using Plots Random.seed!(1) #Define the iterations for the algorithm and the population size -epochs = 1000 -population_size = 1000 +epochs = 100 +population_size = 100 #Number of features which needs to be inserted number_features = 2 @@ -41,7 +41,7 @@ color=:red) #train loss vs validation loss train_validation = plot( - regressor.fitness_history_.train_loss, + [x[1] for x in regressor.fitness_history_.train_loss], label="Training Loss", ylabel="Loss", xlabel="Epoch", @@ -50,7 +50,7 @@ train_validation = plot( plot!( train_validation, - regressor.fitness_history_.val_loss, + [x[1] for x in regressor.fitness_history_.val_loss], label="Validation Loss", linewidth=2 ) diff --git a/test/Main_min_example_mo.jl b/test/Main_min_example_mo.jl index bf53fe7..960e211 100644 --- a/test/Main_min_example_mo.jl +++ b/test/Main_min_example_mo.jl @@ -13,7 +13,7 @@ Random.seed!(1) #Define the iterations for the algorithm and the population size epochs = 1000 -population_size = 10000 +population_size = 1000 #Number of features which needs to be inserted number_features = 2 diff --git a/test/Main_min_with_csv.jl b/test/Main_min_with_csv.jl index b0f2487..f77b55f 100644 --- a/test/Main_min_with_csv.jl +++ b/test/Main_min_with_csv.jl @@ -53,7 +53,7 @@ plot!(pred_vs_actual, vec(y_test), vec(y_test), #train loss vs validation loss train_validation = plot( - regressor.fitness_history_.train_loss, + [x[1] for x in regressor.fitness_history_.train_loss], label="Training Loss", ylabel="Loss", xlabel="Epoch", @@ -62,7 +62,7 @@ train_validation = plot( plot!( train_validation, - regressor.fitness_history_.val_loss, + [x[1] for x in regressor.fitness_history_.val_loss], label="Validation Loss", linewidth=2 ) diff --git a/test/non_dom_sort_test.jl b/test/non_dom_sort_test.jl index 0945987..9bfbba8 100644 --- a/test/non_dom_sort_test.jl +++ b/test/non_dom_sort_test.jl @@ -89,7 +89,7 @@ end (1, 2), (2, 1), (1, 1) ]) - selected2 = selection(pop2) + selected2 = nsga_selection(pop2) @test length(selected2.indices) == 15 @test 15 in selected2.indices # Preserve the best!! alllllways @test length(selected2.fronts) == 5 @@ -100,7 +100,7 @@ end (1, 2, 3), (2, 3, 1), (3, 1, 2), (1, 3, 2), (2, 1, 3), (3, 2, 1) ]) - selected3 = selection(pop3) + selected3 = nsga_selection(pop3) @test length(selected3.indices) == 9 @test 1 in selected3.indices # The best individual should always be selected @test length(selected3.fronts) == 3 From e8ff2657cdb2a70643d94ab82cea9ba6628f1d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Wed, 15 Jan 2025 13:11:30 +1100 Subject: [PATCH 14/41] include tensor function stuff --- src/Entities.jl | 27 ++++++++++++++++++--------- src/GeneExpressionProgramming.jl | 2 +- src/Gep.jl | 14 +++++++------- src/RegressionWrapper.jl | 28 +++++++++++++++++----------- src/Util.jl | 20 ++++++++++++-------- test/regression_wrapper_test.jl | 4 ++-- 6 files changed, 57 insertions(+), 38 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index 1f485c9..24d5cbd 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -89,6 +89,7 @@ include("Util.jl") using .GepUtils using OrderedCollections +using DynamicExpressions """ Toolbox @@ -136,11 +137,12 @@ struct Toolbox fitness_reset::Tuple preamble_syms::Vector{Int8} len_preamble::Int8 + operators_::Union{GenericOperatorEnum,Nothing} function Toolbox(gene_count::Int, head_len::Int, symbols::OrderedDict{Int8,Int8}, gene_connections::Vector{Int8}, callbacks::Dict, nodes::OrderedDict, gep_probs::Dict{String,AbstractFloat}; - unary_prob::Real=0.1, preamble_syms=Int8[], number_of_objectives::Int=1) + unary_prob::Real=0.1, preamble_syms=Int8[], number_of_objectives::Int=1, operators_::Union{GenericOperatorEnum,Nothing}=nothing) fitness_reset= ( ntuple(_ -> Inf, number_of_objectives), @@ -150,10 +152,10 @@ struct Toolbox headsyms = [key for (key, arity) in symbols if arity == 2] unary_syms = [key for (key, arity) in symbols if arity == 1] tailsyms = [key for (key, arity) in symbols if arity < 1 && !(key in preamble_syms)] - len_preamble = length(preamble_syms) == 0 ? 0 : gene_count - gen_start_indices = [gene_count + len_preamble + (gene_len * (i - 1)) for i in 1:gene_count] #depending on the usage should shift everthing + len_preamble = length(preamble_syms) + gen_start_indices = [gene_count + (gene_len * (i - 1)) for i in 1:gene_count] #depending on the usage should shift everthing new(gene_count, head_len, symbols, gene_connections, headsyms, unary_syms, tailsyms, symbols, - callbacks, nodes, gen_start_indices, gep_probs, unary_prob, fitness_reset, preamble_syms, len_preamble) + callbacks, nodes, gen_start_indices, gep_probs, unary_prob, fitness_reset, preamble_syms, len_preamble, operators_) end end @@ -230,12 +232,13 @@ function compile_expression!(chromosome::Chromosome; force_compile::Bool=false) try expression = _karva_raw(chromosome) expression_tree = compile_djl_datatype(expression, chromosome.toolbox.symbols, chromosome.toolbox.callbacks, - chromosome.toolbox.nodes) + chromosome.toolbox.nodes, max(chromosome.toolbox.len_preamble,1)) chromosome.compiled_function = expression_tree chromosome.expression_raw = expression chromosome.fitness = chromosome.toolbox.fitness_reset[2] chromosome.compiled = true - catch + catch e + @error "something went wrong" exception=(e,catch_backtrace()) chromosome.fitness = chromosome.toolbox.fitness_reset[1] end end @@ -315,9 +318,8 @@ Vector{Int8} representing the K-expression of the chromosome function _karva_raw(chromosome::Chromosome) gene_len = chromosome.toolbox.head_len * 2 + 1 gene_count = chromosome.toolbox.gene_count - len_preamble = chromosome.toolbox.len_preamble - connectionsym = @view chromosome.genes[1+len_preamble:gene_count+len_preamble-1] + connectionsym = @view chromosome.genes[1:gene_count-1] genes = chromosome.genes[gene_count:end] arity_gene_ = map(x -> chromosome.toolbox.arrity_by_id[x], genes) @@ -350,7 +352,7 @@ Generate a single gene for GEP. # Returns Vector{Int8} representing gene """ -function generate_gene(headsyms::Vector{Int8}, tailsyms::Vector{Int8}, headlen::Int; unarys::Vector{Int8}=[], unary_prob::Real=0.2) +@inline function generate_gene(headsyms::Vector{Int8}, tailsyms::Vector{Int8}, headlen::Int; unarys::Vector{Int8}=[], unary_prob::Real=0.2) if !isempty(unarys) && rand() < unary_prob heads = vcat(headsyms,tailsyms) push!(heads, rand(unarys)) @@ -364,6 +366,13 @@ function generate_gene(headsyms::Vector{Int8}, tailsyms::Vector{Int8}, headlen:: end +@inline function generate_preamle!(toolbox::Toolbox, preamble::Vector{Int8}) + if !isempty(toolbox.preamble_syms) + append!(preamble, rand(toolbox.preamble_syms, toolbox.gene_count)) + end +end + + """ generate_chromosome(toolbox::Toolbox) diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index f06928d..79d9a9d 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -133,6 +133,6 @@ export GepRegressor, fit! export list_all_functions, list_all_arity, list_all_forward_handlers, list_all_backward_handlers, list_all_genetic_params, set_function!, set_arity!, set_forward_handler!, set_backward_handler!, - update_function! + update_function!, vec_add, vec_mul end diff --git a/src/Gep.jl b/src/Gep.jl index bcd2dc4..9c7084f 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -32,7 +32,7 @@ using GepRegression # Setup parameters toolbox = Toolbox(...) -operators = OperatorEnum(...) +operators = GenericOperatorEnum(...) # Run GEP regression best_solutions, history = runGep( @@ -92,7 +92,7 @@ const Toolbox = GepEntities.Toolbox abstract type EvaluationStrategy end struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy - operators::OperatorEnum + operators::GenericOperatorEnum number_of_objectives::Int x_data::AbstractArray{T} y_data::AbstractArray{T} @@ -104,7 +104,7 @@ struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy penalty::T crash_value::T - function StandardRegressionStrategy{T}(operators::OperatorEnum, + function StandardRegressionStrategy{T}(operators::GenericOperatorEnum, x_data::AbstractArray, y_data::AbstractArray, x_data_test::AbstractArray, @@ -131,13 +131,13 @@ struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy end struct GenericRegressionStrategy <: EvaluationStrategy - operators::OperatorEnum + operators::GenericOperatorEnum number_of_objectives::Int loss_function::Function secOptimizer::Union{Function,Nothing} break_condition::Union{Function,Nothing} - function GenericRegressionStrategy(operators::OperatorEnum, number_of_objectives::Int, loss_function::Function; + function GenericRegressionStrategy(operators::GenericOperatorEnum, number_of_objectives::Int, loss_function::Function; secOptimizer::Union{Function,Nothing}, break_condition::Union{Function,Nothing}) new(operators, number_of_objectives, loss_function, secOptimizer, break_condition) end @@ -146,7 +146,7 @@ end #redesign -> compute fitness should return fitness and crash, we just need to insert the chromosome """ - compute_fitness(elem::Chromosome, operators::OperatorEnum, x_data::AbstractArray{T}, + compute_fitness(elem::Chromosome, operators::GenericOperatorEnum, x_data::AbstractArray{T}, y_data::AbstractArray{T}, loss_function::Function, crash_value::T; validate::Bool=false) where {T<:AbstractFloat} @@ -154,7 +154,7 @@ Computes the fitness score for a chromosome using the specified loss function. # Arguments - `elem::Chromosome`: The chromosome whose fitness needs to be computed -- `operators::OperatorEnum`: The set of mathematical operators available for expression evaluation +- `operators::GenericOperatorEnum`: The set of mathematical operators available for expression evaluation - `x_data::AbstractArray{T}`: Input features for fitness computation - `y_data::AbstractArray{T}`: Target values for fitness computation - `loss_function::Function`: The loss function used to compute fitness diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index e1b89ab..d668e10 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -140,7 +140,7 @@ export fit! export list_all_functions, list_all_arity, list_all_forward_handlers, list_all_backward_handlers, list_all_genetic_params, set_function!, set_arity!, set_forward_handler!, set_backward_handler!, - update_function! + update_function!, vec_add, vec_mul include("Entities.jl") include("Gep.jl") @@ -162,10 +162,14 @@ using .GepUtils using DynamicExpressions using OrderedCollections + const Toolbox = GepRegression.GepEntities.Toolbox const TokenDto = SBPUtils.TokenDto const Chromosome = GepRegression.GepEntities.Chromosome + + + function sqr(x::Vector{T}) where {T<:AbstractFloat} return x .* x end @@ -174,7 +178,6 @@ function sqr(x::T) where {T<:Union{AbstractFloat,Node{<:AbstractFloat}}} return x * x end - """ FUNCTION_LIB_COMMON::Dict{Symbol,Function} @@ -236,7 +239,8 @@ const ARITY_LIB_COMMON = Dict{Symbol,Int8}( :/ => 2, :^ => 2, :min => 2, - :max => 2, :abs => 1, + :max => 2, + :abs => 1, :floor => 1, :ceil => 1, :round => 1, @@ -486,7 +490,7 @@ function create_constants_entries( for elem in entered_terminal_nums utilized_symbols[cur_idx] = 0 - nodes[cur_idx] = parse(node_type, string(elem)) + nodes[cur_idx] = Node{node_type}(;val=parse(node_type, string(elem))) dimension_information[cur_idx] = get(dimensions_to_consider, elem, ZERO_DIM) cur_idx += 1 end @@ -494,7 +498,7 @@ function create_constants_entries( for _ in 1:rnd_count utilized_symbols[cur_idx] = 0 - nodes[cur_idx] = rand() + nodes[cur_idx] = Node{node_type}(;val=rand()) dimension_information[cur_idx] = ZERO_DIM cur_idx += 1 end @@ -503,6 +507,7 @@ function create_constants_entries( end +#preamble syms are just dummies function create_preamble_entries( preamble_syms_raw::Vector{Symbol}, dimensions_to_consider::Dict{Symbol,Vector{Float16}}, @@ -518,7 +523,7 @@ function create_preamble_entries( for elem in preamble_syms_raw utilized_symbols[cur_idx] = 0 - nodes[cur_idx] = Node{AbstractArray}(feature=cur_idx) + nodes[cur_idx] = Node{node_type}(feature=cur_idx) dimension_information[cur_idx] = get(dimensions_to_consider, elem, ZERO_DIM) push!(preamble_syms, cur_idx) cur_idx += 1 @@ -565,7 +570,7 @@ Create a Gene Expression Programming regressor for symbolic regression. """ mutable struct GepRegressor toolbox_::Toolbox - operators_::OperatorEnum + operators_::GenericOperatorEnum dimension_information_::OrderedDict{Int8,Vector{Float16}} best_models_::Union{Nothing,Vector{Chromosome}} fitness_history_::Any @@ -580,8 +585,8 @@ mutable struct GepRegressor considered_dimensions::Dict{Symbol,Vector{Float16}}=Dict{Symbol,Vector{Float16}}(), rnd_count::Int=1, node_type::Type=Float64, - gene_count::Int=2, - head_len::Int=10, + gene_count::Int=3, + head_len::Int=6, preamble_syms::Vector{Symbol}=Symbol[], max_permutations_lib::Int=10000, rounds::Int=4, number_of_objectives::Int=1 @@ -613,7 +618,7 @@ mutable struct GepRegressor dimension_information = merge!(DimensionDict(), feat_dims, const_dims, pre_dims) - operators = OperatorEnum(binary_operators=binary_ops, unary_operators=unary_ops) + operators = GenericOperatorEnum(binary_operators=binary_ops, unary_operators=unary_ops) if !isempty(considered_dimensions) forward_funs, backward_funs, point_ops = create_physical_operations(entered_non_terminals) @@ -637,7 +642,8 @@ mutable struct GepRegressor end toolbox = GepRegression.GepEntities.Toolbox(gene_count, head_len, utilized_symbols, gene_connections_, - callbacks, nodes, GENE_COMMON_PROBS; preamble_syms=preamble_syms, number_of_objectives=number_of_objectives) + callbacks, nodes, GENE_COMMON_PROBS; preamble_syms=preamble_syms_, number_of_objectives=number_of_objectives, + operators_=operators) obj = new() obj.toolbox_ = toolbox diff --git a/src/Util.jl b/src/Util.jl index b265ebb..d5d988c 100644 --- a/src/Util.jl +++ b/src/Util.jl @@ -107,6 +107,10 @@ using Statistics using Random using Base.Threads: @spawn +import Base: +, *, ^ +using DynamicExpressions: @declare_expression_operator + + struct OptimizationHistory{T<:Union{AbstractFloat,Tuple}} train_loss::Vector{T} @@ -417,24 +421,24 @@ result = compile_djl_datatype(rek_string, arity_map, callbacks, nodes) See also: [`DynamicExpressions.Node`](@ref) """ -function compile_djl_datatype(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict) - #just let it fail when it becomes invalid, because then the equation is not that use ful +function compile_djl_datatype(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict, pre_len::Int) + #just let it fail when it becomes invalid, because then the equation is not that use full stack = [] - for elem in reverse(rek_string) + for elem in reverse(rek_string[pre_len:end]) if get(arity_map, elem, 0) == 2 - op1 = (temp = pop!(stack); temp isa Int8 ? nodes[temp] : temp) - op2 = (temp = pop!(stack); temp isa Int8 ? nodes[temp] : temp) + op1 = pop!(stack) + op2 = pop!(stack) ops = callbacks[elem] push!(stack, ops(op1, op2)) elseif get(arity_map, elem, 0) == 1 - op1 = (temp = pop!(stack); temp isa Int8 ? nodes[temp] : temp) + op1 = pop!(stack) ops = callbacks[elem] push!(stack, ops(op1)) else - push!(stack, elem) + push!(stack, elem isa Int8 ? nodes[elem] : elem) end end - return last(stack) + return pre_len==1 ? last(stack) : stack end @inline function retrieve_constants_from_node(node::Node) diff --git a/test/regression_wrapper_test.jl b/test/regression_wrapper_test.jl index 508d150..a3f4ebf 100644 --- a/test/regression_wrapper_test.jl +++ b/test/regression_wrapper_test.jl @@ -51,8 +51,8 @@ Random.seed!(1) @test length(syms) == 4 # 2 constants + 2 random @test all(v -> v == 0, values(syms)) @test length(nodes) == 4 - @test nodes[1] == 1.0 - @test nodes[2] == 2.5 + @test nodes[1] == Node{Float64}(;val=1.0) + @test nodes[2] == Node{Float64}(;val=2.5) end @testset "Physical Operations" begin From af9d54a872ccab20f0622e88b7633330dcd5ff69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Wed, 15 Jan 2025 15:39:14 +1100 Subject: [PATCH 15/41] using formular for tournisize --- src/Gep.jl | 2 +- src/RegressionWrapper.jl | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Gep.jl b/src/Gep.jl index 9c7084f..efdaacc 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -317,7 +317,7 @@ The evolution process stops when either: - Maximum epochs is reached - Break condition specified in evaluation strategy is met => needs to be informed as break_condition(population, epoch) """ -function runGep(epochs::Int, +@inline function runGep(epochs::Int, population_size::Int, toolbox::Toolbox, evalStrategy::EvaluationStrategy; diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index d668e10..8968963 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -685,7 +685,7 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, x_trai optimization_epochs::Int=100, hof::Int=3, loss_fun::Union{String,Function}="mse", correction_epochs::Int=1, correction_amount::Real=0.05, - tourni_size::Int=3, opt_method_const::Symbol=:cg, + opt_method_const::Symbol=:cg, target_dimension::Union{Vector{Float16},Nothing}=nothing, cycles::Int=10, max_iterations::Int=150, n_starts::Int=5 ) @@ -737,7 +737,7 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, x_trai correction_callback=correction_callback, correction_epochs=correction_epochs, correction_amount=correction_amount, - tourni_size=tourni_size, + tourni_size=max(Int(ceil(population_size*0.03)),3), optimization_epochs=optimization_epochs ) @@ -751,7 +751,6 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, loss_f hof::Int=3, correction_epochs::Int=1, correction_amount::Real=0.05, - tourni_size::Int=3, opt_method_const::Symbol=:cg, target_dimension::Union{Vector{Float16},Nothing}=nothing, cycles::Int=10, max_iterations::Int=150, n_starts::Int=5, @@ -799,7 +798,7 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, loss_f correction_callback=correction_callback, correction_epochs=correction_epochs, correction_amount=correction_amount, - tourni_size=tourni_size, + tourni_size=max(Int(ceil(population_size*0.03)),3), optimization_epochs=optimization_epochs ) From 486f98d75b2eee96ab1adf1b13f38a7427f95df0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Wed, 15 Jan 2025 16:49:41 +1100 Subject: [PATCH 16/41] change stuff --- paper/ConstraintViaSBP.jl | 25 ++++++++---- src/Entities.jl | 86 ++++++++++++++++++++++++++++++++++++--- src/RegressionWrapper.jl | 8 ++-- 3 files changed, 100 insertions(+), 19 deletions(-) diff --git a/paper/ConstraintViaSBP.jl b/paper/ConstraintViaSBP.jl index 1434583..b037e2a 100644 --- a/paper/ConstraintViaSBP.jl +++ b/paper/ConstraintViaSBP.jl @@ -12,7 +12,11 @@ using JSON using Statistics -function loss_new(eqn::Node, operators::OperatorEnum, x_data::AbstractArray, y_data::AbstractArray) +function break_condition(population, epoch) + return isclose(mean(population[1].fitness), 0.0) +end + +function loss_new(eqn::Node, operators::GenericOperatorEnum, x_data::AbstractArray, y_data::AbstractArray) try y_pred = eqn(x_data, operators) return get_loss_function("r2_score")(y_data, y_pred) @@ -84,7 +88,7 @@ function main() if case_name in keys(case_data) @show ("Current case: ", case_name) #gep_params - epochs = 100 + epochs = 1000 population_size = 1500 results = DataFrame(Seed=[], @@ -118,25 +122,28 @@ function main() regressor = GepRegressor(num_cols - 1; considered_dimensions=phy_dims, entered_non_terminals=[:+, :-, :*, :/, :sqrt, :sin, :cos, :exp, :log], - max_permutations_lib=10000, rounds=7,number_of_objectives=2) + max_permutations_lib=10000, rounds=7, number_of_objectives=1) + - """next test scenario @inline function loss_new_(elem, validate::Bool) try if isnan(mean(elem.fitness)) || validate y_pred = elem.compiled_function(x_train', regressor.operators_) - return (get_loss_function("mse")(y_train, y_pred), length(elem.expression_raw) * 0.01) + return (get_loss_function("mse")(y_train, y_pred),) else - return (elem.fitness, length(elem.expression_raw) * elem.fitness) + #return (elem.fitness, length(elem.expression_raw) * elem.fitness) + return (elem.fitness,) end catch e - return (typemax(Float64), typemax(Float64)) + return (typemax(Float64),) end end - """ + #perform the regression by entering epochs, population_size, the feature cols, the target col and the loss function - fit!(regressor, epochs, population_size; target_dimension=target_dim) + fit!(regressor, epochs, population_size, x_train', y_train; + x_test=x_test', y_test=y_test', + loss_fun="mse", break_condition=break_condition) end_time = (time_ns() - start_time) / 1e9 elem = regressor.best_models_[1] diff --git a/src/Entities.jl b/src/Entities.jl index 24d5cbd..e284a2f 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -91,6 +91,51 @@ using .GepUtils using OrderedCollections using DynamicExpressions + +""" + Memory Buffer for reducing allocation during runtime! +""" +struct GeneticBuffers + alpha_operator::Vector{Int8} + beta_operator::Vector{Int8} + child_1_genes::Vector{Int8} + child_2_genes::Vector{Int8} + rolled_indices::Vector{Any} + arity_gene::Vector{Int8} +end + + +const THREAD_BUFFERS = let + default_size = 1000 + [GeneticBuffers( + zeros(Int8, default_size), + zeros(Int8, default_size), + Vector{Int8}(undef, default_size), + Vector{Int8}(undef, default_size), + Vector{Any}(undef, default_size), + Vector{Int8}(undef, default_size) + ) for _ in 1:Threads.nthreads()] +end + + +function ensure_buffer_size!(head_len::Int, gene_count::Int) + gene_len = head_len * 2 + 1 + total_gene_length = gene_count - 1 + gene_count * gene_len + + for buffer in THREAD_BUFFERS + if length(buffer.alpha_operator) < total_gene_length + resize!(buffer.alpha_operator, total_gene_length) + resize!(buffer.beta_operator, total_gene_length) + resize!(buffer.child_1_genes, total_gene_length) + resize!(buffer.child_2_genes, total_gene_length) + resize!(buffer.rolled_indices, gene_count + 1) + resize!(buffer.arity_gene, gene_count * gene_len) + end + end +end + + + """ Toolbox @@ -153,7 +198,8 @@ struct Toolbox unary_syms = [key for (key, arity) in symbols if arity == 1] tailsyms = [key for (key, arity) in symbols if arity < 1 && !(key in preamble_syms)] len_preamble = length(preamble_syms) - gen_start_indices = [gene_count + (gene_len * (i - 1)) for i in 1:gene_count] #depending on the usage should shift everthing + gen_start_indices = [gene_count + (gene_len * (i - 1)) for i in 1:gene_count] + ensure_buffer_size!(head_len, gene_count) new(gene_count, head_len, symbols, gene_connections, headsyms, unary_syms, tailsyms, symbols, callbacks, nodes, gen_start_indices, gep_probs, unary_prob, fitness_reset, preamble_syms, len_preamble, operators_) end @@ -227,7 +273,7 @@ Get chromosome's fitness value. # Returns Fitness value or tuple """ -function compile_expression!(chromosome::Chromosome; force_compile::Bool=false) +@inline function compile_expression!(chromosome::Chromosome; force_compile::Bool=false) if !chromosome.compiled || force_compile try expression = _karva_raw(chromosome) @@ -315,7 +361,8 @@ Vector{Int8} representing the K-expression of the chromosome ``` """ -function _karva_raw(chromosome::Chromosome) + +@inline function _karva_raw(chromosome::Chromosome) gene_len = chromosome.toolbox.head_len * 2 + 1 gene_count = chromosome.toolbox.gene_count @@ -336,6 +383,7 @@ function _karva_raw(chromosome::Chromosome) return vcat(rolled_indices...) end + """ generate_gene(headsyms::Vector{Int8}, tailsyms::Vector{Int8}, headlen::Int; unarys::Vector{Int8}=[], unary_prob::Real=0.2) @@ -381,7 +429,7 @@ Generate a new chromosome using toolbox configuration. # Returns New Chromosome instance """ -function generate_chromosome(toolbox::Toolbox) +@inline function generate_chromosome(toolbox::Toolbox) connectors = rand(toolbox.gene_connections, toolbox.gene_count - 1) genes = vcat([generate_gene(toolbox.headsyms, toolbox.tailsyms, toolbox.head_len; unarys=toolbox.unary_syms, unary_prob=toolbox.unary_prob) for _ in 1:toolbox.gene_count]...) @@ -402,14 +450,39 @@ Generate initial population of chromosomes. # Returns Vector of Chromosomes """ -function generate_population(number::Int, toolbox::Toolbox) +@inline function generate_population(number::Int, toolbox::Toolbox) population = Vector{Chromosome}(undef, number) - for i in 1:number + + Threads.@threads for i in 1:number @inbounds population[i] = generate_chromosome(toolbox) end + return population end +""" +@inline function create_operator_masks(gene_seq_alpha::Vector{Int8}, gene_seq_beta::Vector{Int8}, pb::Real=0.2) + buffer = THREAD_BUFFERS[Threads.threadid()] + + fill!(buffer.alpha_operator, 0) + fill!(buffer.beta_operator, 0) + + indices_alpha = rand(1:length(gene_seq_alpha), + min(round(Int, (pb * length(gene_seq_alpha))), + length(gene_seq_alpha))) + indices_beta = rand(1:length(gene_seq_beta), + min(round(Int, (pb * length(gene_seq_beta))), + length(gene_seq_beta))) + + buffer.alpha_operator[indices_alpha] .= Int8(1) + buffer.beta_operator[indices_beta] .= Int8(1) + + return view(buffer.alpha_operator, 1:length(gene_seq_alpha)), + view(buffer.beta_operator, 1:length(gene_seq_beta)) +end + +""" + @inline function create_operator_masks(gene_seq_alpha::Vector{Int8}, gene_seq_beta::Vector{Int8}, pb::Real=0.2) alpha_operator = zeros(Int8, length(gene_seq_alpha)) beta_operator = zeros(Int8, length(gene_seq_beta)) @@ -420,6 +493,7 @@ end return alpha_operator, beta_operator end + @inline function create_operator_point_one_masks(gene_seq_alpha::Vector{Int8}, gene_seq_beta::Vector{Int8}, toolbox::Toolbox) alpha_operator = zeros(Int8, length(gene_seq_alpha)) beta_operator = zeros(Int8, length(gene_seq_beta)) diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index 8968963..d3ebbbd 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -168,8 +168,6 @@ const TokenDto = SBPUtils.TokenDto const Chromosome = GepRegression.GepEntities.Chromosome - - function sqr(x::Vector{T}) where {T<:AbstractFloat} return x .* x end @@ -687,7 +685,8 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, x_trai correction_epochs::Int=1, correction_amount::Real=0.05, opt_method_const::Symbol=:cg, target_dimension::Union{Vector{Float16},Nothing}=nothing, - cycles::Int=10, max_iterations::Int=150, n_starts::Int=5 + cycles::Int=10, max_iterations::Int=150, n_starts::Int=5, + break_condition::Union{Function,Nothing}=nothing ) correction_callback = if !isnothing(target_dimension) @@ -726,7 +725,8 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, x_trai !isnothing(x_test) ? x_test : x_train, !isnothing(y_test) ? y_test : y_train, get_loss_function(loss_fun); - secOptimizer=optimizer_wrapper + secOptimizer=optimizer_wrapper, + break_condition=break_condition ) best, history = runGep(epochs, From 36d79a0f062aa080151fcd717287b78223f90c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Thu, 16 Jan 2025 12:50:33 +1100 Subject: [PATCH 17/41] prior package changes --- src/Entities.jl | 77 +++++++++++++++++++++++--------- src/GeneExpressionProgramming.jl | 5 +-- src/Gep.jl | 61 ++----------------------- src/RegressionWrapper.jl | 5 ++- 4 files changed, 66 insertions(+), 82 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index e284a2f..9cbc3fa 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -4,12 +4,6 @@ A module implementing core data structures and genetic operations for Gene Expression Programming (GEP). -# Core Types -## Symbol Types - depreacated -- `AbstractSymbol`: Base type for GEP symbols -- `BasicSymbol`: Terminal symbols (constants and variables) -- `FunctionalSymbol`: Function symbols with arithmetic operations -- `SymbolConfig`: Configuration container for all symbols ## Evolution Types - `Chromosome`: Individual solution representation @@ -52,13 +46,12 @@ Programming (GEP). # Exports ## Types -- `Chromosome`, `Toolbox`, `AbstractGepToolbox` -- `AbstractSymbol`, `FunctionalSymbol`, `BasicSymbol`, `SymbolConfig` +- `Chromosome`, `Toolbox`, `EvaluationStrategy``, `StandardRegressionStrategy`, `GenericRegressionStrategy` ## Functions ### Core Operations - `fitness`, `set_fitness!` -- `generate_gene`, `generate_preamle!`, `compile_expression!` +- `generate_gene`, `compile_expression!` - `generate_chromosome`, `generate_population` ### Genetic Operations @@ -79,12 +72,11 @@ Programming (GEP). module GepEntities -export Chromosome, Toolbox, AbstractGepToolbox +export Chromosome, Toolbox, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy export fitness, set_fitness! -export generate_gene, generate_preamle!, compile_expression!, generate_chromosome, generate_population +export generate_gene, compile_expression!, generate_chromosome, generate_population export genetic_operations!, replicate, gene_inversion!, gene_mutation!, gene_one_point_cross_over!, gene_two_point_cross_over!, gene_fussion! - include("Util.jl") using .GepUtils @@ -135,6 +127,59 @@ function ensure_buffer_size!(head_len::Int, gene_count::Int) end +abstract type EvaluationStrategy end + +struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy + operators::GenericOperatorEnum + number_of_objectives::Int + x_data::AbstractArray{T} + y_data::AbstractArray{T} + x_data_test::AbstractArray{T} + y_data_test::AbstractArray{T} + loss_function::Function + secOptimizer::Union{Function,Nothing} + break_condition::Union{Function,Nothing} + penalty::T + crash_value::T + + function StandardRegressionStrategy{T}(operators::GenericOperatorEnum, + x_data::AbstractArray, + y_data::AbstractArray, + x_data_test::AbstractArray, + y_data_test::AbstractArray, + loss_function::Function; + secOptimizer::Union{Function,Nothing}=nothing, + break_condition::Union{Function,Nothing}=nothing, + penalty::T=zero(T), + crash_value::T=typemax(T)) where {T<:AbstractFloat} + new(operators, + 1, + x_data, + y_data, + x_data_test, + y_data_test, + loss_function, + secOptimizer, + break_condition, + penalty, + crash_value + ) + end + +end + +struct GenericRegressionStrategy <: EvaluationStrategy + operators::GenericOperatorEnum + number_of_objectives::Int + loss_function::Function + secOptimizer::Union{Function,Nothing} + break_condition::Union{Function,Nothing} + + function GenericRegressionStrategy(operators::GenericOperatorEnum, number_of_objectives::Int, loss_function::Function; + secOptimizer::Union{Function,Nothing}, break_condition::Union{Function,Nothing}) + new(operators, number_of_objectives, loss_function, secOptimizer, break_condition) + end +end """ Toolbox @@ -413,14 +458,6 @@ Vector{Int8} representing gene return vcat(head, tail) end - -@inline function generate_preamle!(toolbox::Toolbox, preamble::Vector{Int8}) - if !isempty(toolbox.preamble_syms) - append!(preamble, rand(toolbox.preamble_syms, toolbox.gene_count)) - end -end - - """ generate_chromosome(toolbox::Toolbox) diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index 79d9a9d..784b32a 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -121,10 +121,9 @@ export get_feature_dims_json, get_target_dim_json, retrieve_coeffs_based_on_simi using .GepEntities -export Chromosome, Toolbox -export AbstractSymbol, FunctionalSymbol, BasicSymbol, SymbolConfig +export Chromosome, Toolbox, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy export fitness, set_fitness! -export generate_gene, generate_preamle!, compile_expression!, generate_chromosome, generate_population +export generate_gene, compile_expression!, generate_chromosome, generate_population export genetic_operations! diff --git a/src/Gep.jl b/src/Gep.jl index efdaacc..49e4afb 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -69,8 +69,6 @@ include("Selection.jl") include("Entities.jl") using .LossFunction -export runGep, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy - using .GepUtils using .EvoSelection using .GepEntities @@ -86,63 +84,12 @@ using Printf using Base.Threads: SpinLock using .Threads -const Chromosome = GepEntities.Chromosome -const Toolbox = GepEntities.Toolbox - -abstract type EvaluationStrategy end - -struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy - operators::GenericOperatorEnum - number_of_objectives::Int - x_data::AbstractArray{T} - y_data::AbstractArray{T} - x_data_test::AbstractArray{T} - y_data_test::AbstractArray{T} - loss_function::Function - secOptimizer::Union{Function,Nothing} - break_condition::Union{Function,Nothing} - penalty::T - crash_value::T - - function StandardRegressionStrategy{T}(operators::GenericOperatorEnum, - x_data::AbstractArray, - y_data::AbstractArray, - x_data_test::AbstractArray, - y_data_test::AbstractArray, - loss_function::Function; - secOptimizer::Union{Function,Nothing}=nothing, - break_condition::Union{Function,Nothing}=nothing, - penalty::T=zero(T), - crash_value::T=typemax(T)) where {T<:AbstractFloat} - new(operators, - 1, - x_data, - y_data, - x_data_test, - y_data_test, - loss_function, - secOptimizer, - break_condition, - penalty, - crash_value - ) - end - -end - -struct GenericRegressionStrategy <: EvaluationStrategy - operators::GenericOperatorEnum - number_of_objectives::Int - loss_function::Function - secOptimizer::Union{Function,Nothing} - break_condition::Union{Function,Nothing} +export runGep - function GenericRegressionStrategy(operators::GenericOperatorEnum, number_of_objectives::Int, loss_function::Function; - secOptimizer::Union{Function,Nothing}, break_condition::Union{Function,Nothing}) - new(operators, number_of_objectives, loss_function, secOptimizer, break_condition) - end -end +const Chromosome = GepEntities.Chromosome +const Toolbox = GepEntities.Toolbox +const EvaluationStrategy = GepEntities.EvaluationStrategy #redesign -> compute fitness should return fitness and crash, we just need to insert the chromosome """ diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index d3ebbbd..a767e72 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -153,7 +153,6 @@ include("Util.jl") using .GepEntities using .LossFunction -using .GepEntities using .EvoSelection using .GepRegression @@ -166,7 +165,9 @@ using OrderedCollections const Toolbox = GepRegression.GepEntities.Toolbox const TokenDto = SBPUtils.TokenDto const Chromosome = GepRegression.GepEntities.Chromosome - +const EvaluationStrategy = GepRegression.GepEntities.EvaluationStrategy +const GenericRegressionStrategy = GepRegression.GepEntities.GenericRegressionStrategy +const StandardRegressionStrategy = GepRegression.GepEntities.StandardRegressionStrategy function sqr(x::Vector{T}) where {T<:AbstractFloat} return x .* x From c33adea1a9cd0d65ad748258b6b25a2a629a9958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Thu, 16 Jan 2025 12:52:22 +1100 Subject: [PATCH 18/41] prior package changes --- src/GeneExpressionProgramming.jl | 151 ++++++++++++++++++++++++------- 1 file changed, 119 insertions(+), 32 deletions(-) diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index 784b32a..0a81112 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -88,50 +88,137 @@ include("Selection.jl") include("Util.jl") include("RegressionWrapper.jl") -export GepEntities, LossFunction, PhysicalConstants, GepUtils, GepRegression, RegressionWrapper +# First export the submodules themselves +export GepEntities, LossFunction, PhysicalConstants, + GepUtils, GepRegression, RegressionWrapper + +# Import GEP core functionality +import .GepRegression: + runGep, + EvaluationStrategy, + StandardRegressionStrategy, + GenericRegressionStrategy + +# Import loss functions +import .LossFunction: + get_loss_function + +# Import utilities +import .GepUtils: + find_indices_with_sum, + compile_djl_datatype, + optimize_constants!, + minmax_scale, + float16_scale, + isclose, + save_state, + load_state, + create_history_recorder, + record_history!, + record!, + close_recorder!, + HistoryRecorder, + OptimizationHistory, + get_history_arrays, + train_test_split + +# Import selection mechanisms +import .EvoSelection: + tournament_selection, + nsga_selection, + dominates_, + fast_non_dominated_sort, + calculate_fronts, + determine_ranks, + assign_crowding_distance + +# Import physical constants functionality +import .PhysicalConstants: + physical_constants, + physical_constants_all, + get_constant, + get_constant_value, + get_constant_dims + +# Import symbolic computation utilities +import .SBPUtils: + TokenLib, + TokenDto, + LibEntry, + TempComputeTree, + create_lib, + create_compute_tree, + propagate_necessary_changes!, + calculate_vector_dimension!, + flush!, + flatten_dependents, + correct_genes!, + equal_unit_forward, + mul_unit_forward, + div_unit_forward, + zero_unit_backward, + zero_unit_forward, + sqr_unit_backward, + sqr_unit_forward, + mul_unit_backward, + div_unit_backward, + equal_unit_backward, + get_feature_dims_json, + get_target_dim_json, + retrieve_coeffs_based_on_similarity + +# Import core GEP entities +import .GepEntities: + Chromosome, + Toolbox, + EvaluationStrategy, + StandardRegressionStrategy, + GenericRegressionStrategy, + fitness, + set_fitness!, + generate_gene, + compile_expression!, + generate_chromosome, + generate_population, + genetic_operations! + +# Import regression wrapper functionality +import .RegressionWrapper: + GepRegressor, + fit!, + list_all_functions, + list_all_arity, + list_all_forward_handlers, + list_all_backward_handlers, + list_all_genetic_params, + set_function!, + set_arity!, + set_forward_handler!, + set_backward_handler!, + update_function!, + vec_add, + vec_mul -using .GepRegression export runGep, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy - -using .LossFunction export get_loss_function - - -using .GepUtils export find_indices_with_sum, compile_djl_datatype, optimize_constants!, minmax_scale, float16_scale, isclose export save_state, load_state, create_history_recorder, record_history!, record!, close_recorder! export HistoryRecorder, OptimizationHistory, get_history_arrays export train_test_split - -using .EvoSelection export tournament_selection, nsga_selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance - - -using .PhysicalConstants -export physical_constants, physical_constants_all -export get_constant, get_constant_value, get_constant_dims - - -using .SBPUtils +export physical_constants, physical_constants_all, get_constant, get_constant_value, get_constant_dims export TokenLib, TokenDto, LibEntry, TempComputeTree -export create_lib, create_compute_tree, propagate_necessary_changes!, calculate_vector_dimension!, flush!, calculate_vector_dimension!, flatten_dependents -export propagate_necessary_changes!, correct_genes! -export equal_unit_forward, mul_unit_forward, div_unit_forward, zero_unit_backward, zero_unit_forward, sqr_unit_backward, sqr_unit_forward, mul_unit_backward, div_unit_backward, equal_unit_backward +export create_lib, create_compute_tree, propagate_necessary_changes!, calculate_vector_dimension!, flush!, flatten_dependents +export correct_genes!, equal_unit_forward, mul_unit_forward, div_unit_forward +export zero_unit_backward, zero_unit_forward, sqr_unit_backward, sqr_unit_forward, mul_unit_backward, div_unit_backward, equal_unit_backward export get_feature_dims_json, get_target_dim_json, retrieve_coeffs_based_on_similarity - - -using .GepEntities -export Chromosome, Toolbox, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy -export fitness, set_fitness! +export Chromosome, Toolbox, fitness, set_fitness! export generate_gene, compile_expression!, generate_chromosome, generate_population export genetic_operations! - - -using .RegressionWrapper export GepRegressor, fit! -export list_all_functions, list_all_arity, list_all_forward_handlers, - list_all_backward_handlers, list_all_genetic_params, - set_function!, set_arity!, set_forward_handler!, set_backward_handler!, - update_function!, vec_add, vec_mul +export list_all_functions, list_all_arity, list_all_forward_handlers +export list_all_backward_handlers, list_all_genetic_params +export set_function!, set_arity!, set_forward_handler!, set_backward_handler! +export update_function!, vec_add, vec_mul end From ed7f9da9bc0b183793df0623e47b09fe08816e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Thu, 16 Jan 2025 13:09:57 +1100 Subject: [PATCH 19/41] correcting imports --- src/GeneExpressionProgramming.jl | 20 ++++++-------------- src/Gep.jl | 10 +++++----- src/RegressionWrapper.jl | 12 ++++++------ 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index 0a81112..9f7f077 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -38,7 +38,6 @@ providing tools for symbolic regression and evolutionary computation. ## Utility Functions - `find_indices_with_sum`, `compile_djl_datatype`: Data processing utilities - `optimize_constants!`: Constants optimization -- `minmax_scale`, `float16_scale`: Scaling functions - `isclose`: Numerical comparison - `save_state`, `load_state`: State persistence - `train_test_split`: Data splitting utility @@ -53,7 +52,7 @@ providing tools for symbolic regression and evolutionary computation. ## History Recording - `HistoryRecorder`, `OptimizationHistory`: History tracking structures -- `create_history_recorder`, `record_history!`, `record!`: Recording functions +- `record_history!`, `record!`: Recording functions - `close_recorder!`, `get_history_arrays`: History management # Example Usage @@ -94,10 +93,7 @@ export GepEntities, LossFunction, PhysicalConstants, # Import GEP core functionality import .GepRegression: - runGep, - EvaluationStrategy, - StandardRegressionStrategy, - GenericRegressionStrategy + runGep # Import loss functions import .LossFunction: @@ -109,11 +105,9 @@ import .GepUtils: compile_djl_datatype, optimize_constants!, minmax_scale, - float16_scale, isclose, save_state, load_state, - create_history_recorder, record_history!, record!, close_recorder!, @@ -195,14 +189,12 @@ import .RegressionWrapper: set_arity!, set_forward_handler!, set_backward_handler!, - update_function!, - vec_add, - vec_mul + update_function! export runGep, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy export get_loss_function -export find_indices_with_sum, compile_djl_datatype, optimize_constants!, minmax_scale, float16_scale, isclose -export save_state, load_state, create_history_recorder, record_history!, record!, close_recorder! +export find_indices_with_sum, compile_djl_datatype, optimize_constants!, minmax_scale, isclose +export save_state, load_state, record_history!, record!, close_recorder! export HistoryRecorder, OptimizationHistory, get_history_arrays export train_test_split export tournament_selection, nsga_selection, dominates_, fast_non_dominated_sort, calculate_fronts, determine_ranks, assign_crowding_distance @@ -219,6 +211,6 @@ export GepRegressor, fit! export list_all_functions, list_all_arity, list_all_forward_handlers export list_all_backward_handlers, list_all_genetic_params export set_function!, set_arity!, set_forward_handler!, set_backward_handler! -export update_function!, vec_add, vec_mul +export update_function! end diff --git a/src/Gep.jl b/src/Gep.jl index 49e4afb..9d99dc0 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -66,12 +66,12 @@ module GepRegression include("Losses.jl") include("Util.jl") include("Selection.jl") -include("Entities.jl") +#include("Entities.jl") using .LossFunction using .GepUtils using .EvoSelection -using .GepEntities +using ..GepEntities using Random using Statistics @@ -87,9 +87,9 @@ using .Threads export runGep -const Chromosome = GepEntities.Chromosome -const Toolbox = GepEntities.Toolbox -const EvaluationStrategy = GepEntities.EvaluationStrategy +#const Chromosome = GepEntities.Chromosome +#const Toolbox = GepEntities.Toolbox +#const EvaluationStrategy = GepEntities.EvaluationStrategy #redesign -> compute fitness should return fitness and crash, we just need to insert the chromosome """ diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index a767e72..f1b1584 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -162,12 +162,12 @@ using DynamicExpressions using OrderedCollections -const Toolbox = GepRegression.GepEntities.Toolbox -const TokenDto = SBPUtils.TokenDto -const Chromosome = GepRegression.GepEntities.Chromosome -const EvaluationStrategy = GepRegression.GepEntities.EvaluationStrategy -const GenericRegressionStrategy = GepRegression.GepEntities.GenericRegressionStrategy -const StandardRegressionStrategy = GepRegression.GepEntities.StandardRegressionStrategy +#const Toolbox = GepRegression.GepEntities.Toolbox +#const TokenDto = SBPUtils.TokenDto +#const Chromosome = GepRegression.GepEntities.Chromosome +#const EvaluationStrategy = GepRegression.GepEntities.EvaluationStrategy +#const GenericRegressionStrategy = GepRegression.GepEntities.GenericRegressionStrategy +#const StandardRegressionStrategy = GepRegression.GepEntities.StandardRegressionStrategy function sqr(x::Vector{T}) where {T<:AbstractFloat} return x .* x From e7281abab63ba69b0a1ef38a06d59f836f01a52d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Thu, 16 Jan 2025 17:12:47 +1100 Subject: [PATCH 20/41] correcting imports --- Project.toml | 3 +-- src/Entities.jl | 1 - src/GeneExpressionProgramming.jl | 21 ++++++++++++++++++++- src/Gep.jl | 11 ++++++++++- src/RegressionWrapper.jl | 2 +- 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Project.toml b/Project.toml index 57d393d..03a5834 100644 --- a/Project.toml +++ b/Project.toml @@ -3,12 +3,12 @@ uuid = "2f0a5bb0-5f4f-4f7f-b515-93a1e67611af" authors = ["Max Reissmann "] version = "0.3.4" - [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" DynamicExpressions = "a40a106e-89c9-4ca8-8020-a735e8728b6b" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" @@ -28,7 +28,6 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" - [compat] BenchmarkTools = "1" CSV = "0.10" diff --git a/src/Entities.jl b/src/Entities.jl index 9cbc3fa..80a3a5c 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -126,7 +126,6 @@ function ensure_buffer_size!(head_len::Int, gene_count::Int) end end - abstract type EvaluationStrategy end struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index 9f7f077..4bc3d22 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -85,11 +85,12 @@ include("PhyConstants.jl") include("Sbp.jl") include("Selection.jl") include("Util.jl") +include("Surrogate.jl") include("RegressionWrapper.jl") # First export the submodules themselves export GepEntities, LossFunction, PhysicalConstants, - GepUtils, GepRegression, RegressionWrapper + GepUtils, GepRegression, RegressionWrapper, GPSurrogate # Import GEP core functionality import .GepRegression: @@ -191,6 +192,20 @@ import .RegressionWrapper: set_backward_handler!, update_function! +import .GPSurrogate: + GPRegressionStrategy, + ExpectedImprovement, + PropabilityImrpovement, + EntropySearch, + GPState + update_surrogate!, + predict_gp, + compute_acquisition, + add_point! + ExpectedImprovement, + PropabilityImrpovement, + EntropySearch + export runGep, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy export get_loss_function export find_indices_with_sum, compile_djl_datatype, optimize_constants!, minmax_scale, isclose @@ -212,5 +227,9 @@ export list_all_functions, list_all_arity, list_all_forward_handlers export list_all_backward_handlers, list_all_genetic_params export set_function!, set_arity!, set_forward_handler!, set_backward_handler! export update_function! +export GPRegressionStrategy, ExpectedImprovement, PropabilityImrpovement, EntropySearch, GPState +export update_surrogate!, predict_gp, compute_acquisition, add_point! +export ExpectedImprovement, PropabilityImrpovement, EntropySearch + end diff --git a/src/Gep.jl b/src/Gep.jl index 9d99dc0..95fd81b 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -66,7 +66,6 @@ module GepRegression include("Losses.jl") include("Util.jl") include("Selection.jl") -#include("Entities.jl") using .LossFunction using .GepUtils @@ -133,6 +132,10 @@ end return evalArgs.loss_function(elem, validate) end +@inline function compute_fitness(elem::Chromosome, evalArgs::GenericRegressionStrategy; validate::Bool=false) + return evalArgs.loss_function(elem, validate) +end + """ perform_step!(population::Vector{Chromosome}, parents::Vector{Chromosome}, next_gen::Vector{Chromosome}, toolbox::Toolbox, mating_size::Int) @@ -178,6 +181,10 @@ Performs one evolutionary step in the GEP algorithm, creating and evaluating new end +function update_surrogate!(::EvaluationStrategy) + nothing +end + """ perform_correction_callback!(population::Vector{Chromosome}, epoch::Int, correction_epochs::Int, correction_amount::Real, @@ -327,6 +334,8 @@ The evolution process stops when either: (:validation_loss, @sprintf("%.6e", mean(val_loss))) ]) + update_surrogate!(evalStrategy) + if !isnothing(evalStrategy.break_condition) && evalStrategy.break_condition(population, epoch) break diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index f1b1584..866f038 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -161,7 +161,7 @@ using .GepUtils using DynamicExpressions using OrderedCollections - +# #const Toolbox = GepRegression.GepEntities.Toolbox #const TokenDto = SBPUtils.TokenDto #const Chromosome = GepRegression.GepEntities.Chromosome From bceb13ea861cbfc5f21c16608f5954d134cbbc27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Thu, 16 Jan 2025 22:02:42 +1100 Subject: [PATCH 21/41] sg --- src/GeneExpressionProgramming.jl | 6 +++--- src/Gep.jl | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index 4bc3d22..af114ac 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -197,13 +197,13 @@ import .GPSurrogate: ExpectedImprovement, PropabilityImrpovement, EntropySearch, - GPState + GPState, update_surrogate!, predict_gp, compute_acquisition, - add_point! + add_point!, ExpectedImprovement, - PropabilityImrpovement, + PropabilityImprovement, EntropySearch export runGep, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy diff --git a/src/Gep.jl b/src/Gep.jl index 95fd81b..3692359 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -67,10 +67,15 @@ include("Losses.jl") include("Util.jl") include("Selection.jl") +include("Surrogate.jl") + + using .LossFunction using .GepUtils using .EvoSelection using ..GepEntities +using .GPSurrogate + using Random using Statistics @@ -132,7 +137,7 @@ end return evalArgs.loss_function(elem, validate) end -@inline function compute_fitness(elem::Chromosome, evalArgs::GenericRegressionStrategy; validate::Bool=false) +@inline function compute_fitness(elem::Chromosome, evalArgs::GPRegressionStrategy; validate::Bool=false) return evalArgs.loss_function(elem, validate) end @@ -181,7 +186,7 @@ Performs one evolutionary step in the GEP algorithm, creating and evaluating new end -function update_surrogate!(::EvaluationStrategy) +@inline function update_surrogate!(::EvaluationStrategy) nothing end From 5afd2a51f9c4860199c61f97c2e615786ee5d35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Fri, 17 Jan 2025 12:20:28 +1100 Subject: [PATCH 22/41] change adress --- src/Entities.jl | 12 ++++++------ src/GeneExpressionProgramming.jl | 4 ++-- src/Gep.jl | 6 +++--- src/RegressionWrapper.jl | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index 80a3a5c..70e0128 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -129,7 +129,7 @@ end abstract type EvaluationStrategy end struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy - operators::GenericOperatorEnum + operators::OperatorEnum number_of_objectives::Int x_data::AbstractArray{T} y_data::AbstractArray{T} @@ -141,7 +141,7 @@ struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy penalty::T crash_value::T - function StandardRegressionStrategy{T}(operators::GenericOperatorEnum, + function StandardRegressionStrategy{T}(operators::OperatorEnum, x_data::AbstractArray, y_data::AbstractArray, x_data_test::AbstractArray, @@ -168,13 +168,13 @@ struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy end struct GenericRegressionStrategy <: EvaluationStrategy - operators::GenericOperatorEnum + operators::OperatorEnum number_of_objectives::Int loss_function::Function secOptimizer::Union{Function,Nothing} break_condition::Union{Function,Nothing} - function GenericRegressionStrategy(operators::GenericOperatorEnum, number_of_objectives::Int, loss_function::Function; + function GenericRegressionStrategy(operators::OperatorEnum, number_of_objectives::Int, loss_function::Function; secOptimizer::Union{Function,Nothing}, break_condition::Union{Function,Nothing}) new(operators, number_of_objectives, loss_function, secOptimizer, break_condition) end @@ -226,12 +226,12 @@ struct Toolbox fitness_reset::Tuple preamble_syms::Vector{Int8} len_preamble::Int8 - operators_::Union{GenericOperatorEnum,Nothing} + operators_::Union{OperatorEnum,Nothing} function Toolbox(gene_count::Int, head_len::Int, symbols::OrderedDict{Int8,Int8}, gene_connections::Vector{Int8}, callbacks::Dict, nodes::OrderedDict, gep_probs::Dict{String,AbstractFloat}; - unary_prob::Real=0.1, preamble_syms=Int8[], number_of_objectives::Int=1, operators_::Union{GenericOperatorEnum,Nothing}=nothing) + unary_prob::Real=0.1, preamble_syms=Int8[], number_of_objectives::Int=1, operators_::Union{OperatorEnum,Nothing}=nothing) fitness_reset= ( ntuple(_ -> Inf, number_of_objectives), diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index af114ac..584b65f 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -227,9 +227,9 @@ export list_all_functions, list_all_arity, list_all_forward_handlers export list_all_backward_handlers, list_all_genetic_params export set_function!, set_arity!, set_forward_handler!, set_backward_handler! export update_function! -export GPRegressionStrategy, ExpectedImprovement, PropabilityImrpovement, EntropySearch, GPState +export GPRegressionStrategy, ExpectedImprovement, PropabilityImprovement, EntropySearch, GPState export update_surrogate!, predict_gp, compute_acquisition, add_point! -export ExpectedImprovement, PropabilityImrpovement, EntropySearch + end diff --git a/src/Gep.jl b/src/Gep.jl index 3692359..956dc15 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -32,7 +32,7 @@ using GepRegression # Setup parameters toolbox = Toolbox(...) -operators = GenericOperatorEnum(...) +operators = OperatorEnum(...) # Run GEP regression best_solutions, history = runGep( @@ -97,7 +97,7 @@ export runGep #redesign -> compute fitness should return fitness and crash, we just need to insert the chromosome """ - compute_fitness(elem::Chromosome, operators::GenericOperatorEnum, x_data::AbstractArray{T}, + compute_fitness(elem::Chromosome, operators::OperatorEnum, x_data::AbstractArray{T}, y_data::AbstractArray{T}, loss_function::Function, crash_value::T; validate::Bool=false) where {T<:AbstractFloat} @@ -105,7 +105,7 @@ Computes the fitness score for a chromosome using the specified loss function. # Arguments - `elem::Chromosome`: The chromosome whose fitness needs to be computed -- `operators::GenericOperatorEnum`: The set of mathematical operators available for expression evaluation +- `operators::OperatorEnum`: The set of mathematical operators available for expression evaluation - `x_data::AbstractArray{T}`: Input features for fitness computation - `y_data::AbstractArray{T}`: Target values for fitness computation - `loss_function::Function`: The loss function used to compute fitness diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index 866f038..62aa7d9 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -569,7 +569,7 @@ Create a Gene Expression Programming regressor for symbolic regression. """ mutable struct GepRegressor toolbox_::Toolbox - operators_::GenericOperatorEnum + operators_::OperatorEnum dimension_information_::OrderedDict{Int8,Vector{Float16}} best_models_::Union{Nothing,Vector{Chromosome}} fitness_history_::Any @@ -617,7 +617,7 @@ mutable struct GepRegressor dimension_information = merge!(DimensionDict(), feat_dims, const_dims, pre_dims) - operators = GenericOperatorEnum(binary_operators=binary_ops, unary_operators=unary_ops) + operators = OperatorEnum(binary_operators=binary_ops, unary_operators=unary_ops) if !isempty(considered_dimensions) forward_funs, backward_funs, point_ops = create_physical_operations(entered_non_terminals) From b4eb411a296b7bb1f650cbdbc17dfd62a1874348 Mon Sep 17 00:00:00 2001 From: maxreiss123 <32074940+maxreiss123@users.noreply.github.com> Date: Fri, 17 Jan 2025 12:23:38 +1100 Subject: [PATCH 23/41] Update GeneExpressionProgramming.jl --- src/GeneExpressionProgramming.jl | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index 584b65f..ecf9152 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -85,7 +85,6 @@ include("PhyConstants.jl") include("Sbp.jl") include("Selection.jl") include("Util.jl") -include("Surrogate.jl") include("RegressionWrapper.jl") # First export the submodules themselves @@ -192,19 +191,7 @@ import .RegressionWrapper: set_backward_handler!, update_function! -import .GPSurrogate: - GPRegressionStrategy, - ExpectedImprovement, - PropabilityImrpovement, - EntropySearch, - GPState, - update_surrogate!, - predict_gp, - compute_acquisition, - add_point!, - ExpectedImprovement, - PropabilityImprovement, - EntropySearch + export runGep, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy export get_loss_function @@ -227,8 +214,6 @@ export list_all_functions, list_all_arity, list_all_forward_handlers export list_all_backward_handlers, list_all_genetic_params export set_function!, set_arity!, set_forward_handler!, set_backward_handler! export update_function! -export GPRegressionStrategy, ExpectedImprovement, PropabilityImprovement, EntropySearch, GPState -export update_surrogate!, predict_gp, compute_acquisition, add_point! From 6802b5b50403573397d951a488e353098f033664 Mon Sep 17 00:00:00 2001 From: maxreiss123 <32074940+maxreiss123@users.noreply.github.com> Date: Fri, 17 Jan 2025 12:24:02 +1100 Subject: [PATCH 24/41] Update Gep.jl --- src/Gep.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Gep.jl b/src/Gep.jl index 956dc15..3a7f31c 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -67,14 +67,11 @@ include("Losses.jl") include("Util.jl") include("Selection.jl") -include("Surrogate.jl") - using .LossFunction using .GepUtils using .EvoSelection using ..GepEntities -using .GPSurrogate using Random @@ -137,9 +134,6 @@ end return evalArgs.loss_function(elem, validate) end -@inline function compute_fitness(elem::Chromosome, evalArgs::GPRegressionStrategy; validate::Bool=false) - return evalArgs.loss_function(elem, validate) -end """ perform_step!(population::Vector{Chromosome}, parents::Vector{Chromosome}, From 4cefcedf5ea130bb1bd3ff2d29dca8ad42f22085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Fri, 17 Jan 2025 13:10:13 +1100 Subject: [PATCH 25/41] reset changes --- src/Util.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Util.jl b/src/Util.jl index d5d988c..31fe4b0 100644 --- a/src/Util.jl +++ b/src/Util.jl @@ -107,9 +107,6 @@ using Statistics using Random using Base.Threads: @spawn -import Base: +, *, ^ -using DynamicExpressions: @declare_expression_operator - struct OptimizationHistory{T<:Union{AbstractFloat,Tuple}} From cc5618d280a348ae0fbc11ee7b25b557a2c93706 Mon Sep 17 00:00:00 2001 From: maxreiss123 <32074940+maxreiss123@users.noreply.github.com> Date: Sat, 18 Jan 2025 10:57:04 +1100 Subject: [PATCH 26/41] Update Gep.jl --- src/Gep.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Gep.jl b/src/Gep.jl index 3a7f31c..8645705 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -316,7 +316,7 @@ The evolution process stops when either: fits_representation[index] = population[index].fitness end - if !isnothing(evalStrategy.secOptimizer) && epochs % optimization_epochs == 0 && population[1].fitness < prev_best + if !isnothing(evalStrategy.secOptimizer) && epoch % optimization_epochs == 0 && population[1].fitness < prev_best evalStrategy.secOptimizer(population) fits_representation[1] = population[1].fitness prev_best = fits_representation[1] From b361e1bc2bdea03331b0d8d0725eddc095abef55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sat, 18 Jan 2025 11:18:24 +1100 Subject: [PATCH 27/41] add further changes --- Project.toml | 1 + paper/ConstraintViaSBP.jl | 8 +- src/Entities.jl | 172 ++++++++++++++++---------------------- src/Losses.jl | 50 +++++++++-- test/Entity_test.jl | 47 ----------- 5 files changed, 119 insertions(+), 159 deletions(-) diff --git a/Project.toml b/Project.toml index 03a5834..580ff2f 100644 --- a/Project.toml +++ b/Project.toml @@ -17,6 +17,7 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +LoopVectorization = "bdcacae8-1622-11e9-2a5c-532679323890" Optim = "429524aa-4258-5aef-a3af-852621145aeb" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" diff --git a/paper/ConstraintViaSBP.jl b/paper/ConstraintViaSBP.jl index b037e2a..37af476 100644 --- a/paper/ConstraintViaSBP.jl +++ b/paper/ConstraintViaSBP.jl @@ -16,7 +16,7 @@ function break_condition(population, epoch) return isclose(mean(population[1].fitness), 0.0) end -function loss_new(eqn::Node, operators::GenericOperatorEnum, x_data::AbstractArray, y_data::AbstractArray) +function loss_new(eqn::Node, operators::OperatorEnum, x_data::AbstractArray, y_data::AbstractArray) try y_pred = eqn(x_data, operators) return get_loss_function("r2_score")(y_data, y_pred) @@ -131,11 +131,11 @@ function main() y_pred = elem.compiled_function(x_train', regressor.operators_) return (get_loss_function("mse")(y_train, y_pred),) else - #return (elem.fitness, length(elem.expression_raw) * elem.fitness) - return (elem.fitness,) + return (elem.fitness, length(elem.expression_raw) * elem.fitness) + #return (elem.fitness,) end catch e - return (typemax(Float64),) + return (typemax(Float64),typemax(Float64)) end end diff --git a/src/Entities.jl b/src/Entities.jl index 70e0128..d999f78 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -496,43 +496,29 @@ Vector of Chromosomes return population end -""" + @inline function create_operator_masks(gene_seq_alpha::Vector{Int8}, gene_seq_beta::Vector{Int8}, pb::Real=0.2) buffer = THREAD_BUFFERS[Threads.threadid()] - - fill!(buffer.alpha_operator, 0) - fill!(buffer.beta_operator, 0) - - indices_alpha = rand(1:length(gene_seq_alpha), - min(round(Int, (pb * length(gene_seq_alpha))), - length(gene_seq_alpha))) - indices_beta = rand(1:length(gene_seq_beta), - min(round(Int, (pb * length(gene_seq_beta))), - length(gene_seq_beta))) - - buffer.alpha_operator[indices_alpha] .= Int8(1) - buffer.beta_operator[indices_beta] .= Int8(1) - - return view(buffer.alpha_operator, 1:length(gene_seq_alpha)), - view(buffer.beta_operator, 1:length(gene_seq_beta)) -end + len_a = length(gene_seq_alpha) -""" + buffer.alpha_operator[1:len_a] .= zeros(Int8) + buffer.beta_operator[1:len_a] .= zeros(Int8) -@inline function create_operator_masks(gene_seq_alpha::Vector{Int8}, gene_seq_beta::Vector{Int8}, pb::Real=0.2) - alpha_operator = zeros(Int8, length(gene_seq_alpha)) - beta_operator = zeros(Int8, length(gene_seq_beta)) - indices_alpha = rand(1:length(gene_seq_alpha), min(round(Int, (pb * length(gene_seq_alpha))), length(gene_seq_alpha))) - indices_beta = rand(1:length(gene_seq_beta), min(round(Int, (pb * length(gene_seq_beta))), length(gene_seq_beta))) - alpha_operator[indices_alpha] .= Int8(1) - beta_operator[indices_beta] .= Int8(1) - return alpha_operator, beta_operator + indices_alpha = rand(1:len_a, min(round(Int, (pb * len_a)), len_a)) + indices_beta = rand(1:len_a, min(round(Int, (pb * len_a)), len_a)) + + buffer.alpha_operator[indices_alpha] .= Int8(1) + buffer.beta_operator[indices_beta] .= Int8(1) end @inline function create_operator_point_one_masks(gene_seq_alpha::Vector{Int8}, gene_seq_beta::Vector{Int8}, toolbox::Toolbox) - alpha_operator = zeros(Int8, length(gene_seq_alpha)) - beta_operator = zeros(Int8, length(gene_seq_beta)) + buffer = THREAD_BUFFERS[Threads.threadid()] + len_a = length(gene_seq_alpha) + + buffer.alpha_operator[1:len_a] .= zeros(Int8) + buffer.beta_operator[1:len_a] .= zeros(Int8) + head_len = toolbox.head_len gene_len = head_len * 2 + 1 @@ -542,20 +528,21 @@ end point1 = rand(ref:mid) point2 = rand((mid+1):(ref+gene_len-1)) - alpha_operator[point1:point2] .= Int8(1) + buffer.alpha_operator[point1:point2] .= Int8(1) point1 = rand(ref:mid) point2 = rand((mid+1):(ref+gene_len-1)) - beta_operator[point1:point2] .= Int8(1) + buffer.beta_operator[point1:point2] .= Int8(1) end - - return alpha_operator, beta_operator end @inline function create_operator_point_two_masks(gene_seq_alpha::Vector{Int8}, gene_seq_beta::Vector{Int8}, toolbox::Toolbox) - alpha_operator = zeros(Int8, length(gene_seq_alpha)) - beta_operator = zeros(Int8, length(gene_seq_beta)) + buffer = THREAD_BUFFERS[Threads.threadid()] + len_a = length(gene_seq_alpha) + + buffer.alpha_operator[1:len_a] .= zeros(Int8) + buffer.beta_operator[1:len_a] .= zeros(Int8) head_len = toolbox.head_len gene_len = head_len * 2 + 1 @@ -569,17 +556,17 @@ end point1 = rand(start:quarter) point2 = rand(quarter+1:half) point3 = rand(half+1:end_gene) - alpha_operator[point1:point2] .= Int8(1) - alpha_operator[point3:end_gene] .= Int8(1) + buffer.alpha_operator[point1:point2] .= Int8(1) + buffer.alpha_operator[point3:end_gene] .= Int8(1) point1 = rand(start:end_gene) point2 = rand(point1:end_gene) - beta_operator[point1:point2] .= Int8(1) - beta_operator[point2+1:end_gene] .= Int8(1) + buffer.beta_operator[point1:point2] .= Int8(1) + buffer.beta_operator[point2+1:end_gene] .= Int8(1) end - return alpha_operator, beta_operator + end @inline function replicate(chromosome1::Chromosome, chromosome2::Chromosome, toolbox) @@ -588,97 +575,82 @@ end @inline function gene_dominant_fusion!(chromosome1::Chromosome, chromosome2::Chromosome, pb::Real=0.2) - gene_seq_alpha = chromosome1.genes - gene_seq_beta = chromosome2.genes - alpha_operator, beta_operator = create_operator_masks(gene_seq_alpha, gene_seq_beta, pb) - - child_1_genes = similar(gene_seq_alpha) - child_2_genes = similar(gene_seq_beta) + buffer = THREAD_BUFFERS[Threads.threadid()] + len_a =length(chromosome1.genes) + create_operator_masks(chromosome1.genes, chromosome2.genes, pb) - @inbounds @simd for i in eachindex(gene_seq_alpha) - child_1_genes[i] = alpha_operator[i] == 1 ? max(gene_seq_alpha[i], gene_seq_beta[i]) : gene_seq_alpha[i] - child_2_genes[i] = beta_operator[i] == 1 ? max(gene_seq_alpha[i], gene_seq_beta[i]) : gene_seq_beta[i] + @inbounds @simd for i in eachindex(chromosome1.genes) + buffer.child_1_genes[i] = buffer.alpha_operator[i] == 1 ? max(chromosome1.genes[i], chromosome2.genes[i]) : chromosome1.genes[i] + buffer.child_2_genes[i] = buffer.beta_operator[i] == 1 ? max(chromosome1.genes[i], chromosome2.genes[i]) : chromosome2.genes[i] end - chromosome1.genes = child_1_genes - chromosome2.genes = child_2_genes + chromosome1.genes .= @view buffer.child_1_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gen_rezessiv!(chromosome1::Chromosome, chromosome2::Chromosome, pb::Real=0.2) - gene_seq_alpha = chromosome1.genes - gene_seq_beta = chromosome2.genes - alpha_operator, beta_operator = create_operator_masks(gene_seq_alpha, gene_seq_beta, pb) - - child_1_genes = similar(gene_seq_alpha) - child_2_genes = similar(gene_seq_beta) + buffer = THREAD_BUFFERS[Threads.threadid()] + len_a =length(chromosome1.genes) + create_operator_masks(chromosome1.genes, chromosome2.genes, pb) - @inbounds @simd for i in eachindex(gene_seq_alpha) - child_1_genes[i] = alpha_operator[i] == 1 ? min(gene_seq_alpha[i], gene_seq_beta[i]) : gene_seq_alpha[i] - child_2_genes[i] = beta_operator[i] == 1 ? min(gene_seq_alpha[i], gene_seq_beta[i]) : gene_seq_beta[i] + @inbounds @simd for i in eachindex(chromosome1.genes) + buffer.child_1_genes[i] = buffer.alpha_operator[i] == 1 ? min(chromosome1.genes[i], chromosome2.genes[i]) : chromosome1.genes[i] + buffer.child_2_genes[i] = buffer.beta_operator[i] == 1 ? min(chromosome1.genes[i], chromosome2.genes[i]) : chromosome2.genes[i] end - chromosome1.genes = child_1_genes - chromosome2.genes = child_2_genes + chromosome1.genes .= @view buffer.child_1_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gene_fussion!(chromosome1::Chromosome, chromosome2::Chromosome, pb::Real=0.2) - gene_seq_alpha = chromosome1.genes - gene_seq_beta = chromosome2.genes - alpha_operator, beta_operator = create_operator_masks(gene_seq_alpha, gene_seq_beta, pb) - - child_1_genes = similar(gene_seq_alpha) - child_2_genes = similar(gene_seq_beta) + buffer = THREAD_BUFFERS[Threads.threadid()] + len_a = length(chromosome1.genes) + create_operator_masks(chromosome1.genes, chromosome2.genes, pb) - @inbounds @simd for i in eachindex(gene_seq_alpha) - child_1_genes[i] = alpha_operator[i] == 1 ? Int8((gene_seq_alpha[i] + gene_seq_beta[i]) ÷ 2) : gene_seq_alpha[i] - child_2_genes[i] = beta_operator[i] == 1 ? Int8((gene_seq_alpha[i] + gene_seq_beta[i]) ÷ 2) : gene_seq_beta[i] + @inbounds @simd for i in eachindex(chromosome1.genes) + buffer.child_1_genes[i] = buffer.alpha_operator[i] == 1 ? Int8((chromosome1.genes[i] + chromosome2.genes[i]) ÷ 2) : chromosome1.genes[i] + buffer.child_2_genes[i] = buffer.beta_operator[i] == 1 ? Int8((chromosome1.genes[i] + chromosome2.genes[i]) ÷ 2) : chromosome2.genes[i] end - chromosome1.genes = child_1_genes - chromosome2.genes = child_2_genes + chromosome1.genes .= @view buffer.child_1_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gene_one_point_cross_over!(chromosome1::Chromosome, chromosome2::Chromosome) - gene_seq_alpha = chromosome1.genes - gene_seq_beta = chromosome2.genes - alpha_operator, beta_operator = create_operator_point_one_masks(gene_seq_alpha, gene_seq_beta, chromosome1.toolbox) - - child_1_genes = similar(gene_seq_alpha) - child_2_genes = similar(gene_seq_beta) + buffer = THREAD_BUFFERS[Threads.threadid()] + len_a = length(chromosome1.genes) + create_operator_point_one_masks(chromosome1.genes, chromosome2.genes, chromosome1.toolbox) - @inbounds @simd for i in eachindex(gene_seq_alpha) - child_1_genes[i] = alpha_operator[i] == 1 ? gene_seq_alpha[i] : gene_seq_beta[i] - child_2_genes[i] = beta_operator[i] == 1 ? gene_seq_beta[i] : gene_seq_alpha[i] + @inbounds @simd for i in eachindex(chromosome1.genes) + buffer.child_1_genes[i] = buffer.alpha_operator[i] == 1 ? chromosome1.genes[i] : chromosome2.genes[i] + buffer.child_2_genes[i] = buffer.beta_operator[i] == 1 ? chromosome2.genes[i] : chromosome1.genes[i] end - chromosome1.genes = child_1_genes - chromosome2.genes = child_2_genes + chromosome1.genes .= @view buffer.child_1_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gene_two_point_cross_over!(chromosome1::Chromosome, chromosome2::Chromosome) - gene_seq_alpha = chromosome1.genes - gene_seq_beta = chromosome2.genes - alpha_operator, beta_operator = create_operator_point_two_masks(gene_seq_alpha, gene_seq_beta, chromosome1.toolbox) - - child_1_genes = similar(gene_seq_alpha) - child_2_genes = similar(gene_seq_beta) + buffer = THREAD_BUFFERS[Threads.threadid()] + len_a = length(chromosome1.genes) + create_operator_point_two_masks(chromosome1.genes, chromosome2.genes, chromosome1.toolbox) - @inbounds @simd for i in eachindex(gene_seq_alpha) - child_1_genes[i] = alpha_operator[i] == 1 ? gene_seq_alpha[i] : gene_seq_beta[i] - child_2_genes[i] = beta_operator[i] == 1 ? gene_seq_beta[i] : gene_seq_alpha[i] + @inbounds @simd for i in eachindex(chromosome1.genes) + buffer.child_1_genes[i] = buffer.alpha_operator[i] == 1 ? chromosome1.genes[i] : chromosome2.genes[i] + buffer.child_2_genes[i] = buffer.beta_operator[i] == 1 ? chromosome2.genes[i] : chromosome1.genes[i] end - chromosome1.genes = child_1_genes - chromosome2.genes = child_2_genes + chromosome1.genes .= @view buffer.child_1_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gene_mutation!(chromosome1::Chromosome, pb::Real=0.25) - gene_seq_alpha = chromosome1.genes - alpha_operator, _ = create_operator_masks(gene_seq_alpha, gene_seq_alpha, pb) - mutation_seq_1 = generate_chromosome(chromosome1.toolbox) + buffer = THREAD_BUFFERS[Threads.threadid()] + create_operator_masks(chromosome1.genes, chromosome1.genes, pb) + buffer.child_1_genes[1:length(chromosome1.genes)] .= generate_chromosome(chromosome1.toolbox).genes[1:length(chromosome1.genes)] - @inbounds @simd for i in eachindex(gene_seq_alpha) - gene_seq_alpha[i] = alpha_operator[i] == 1 ? mutation_seq_1.genes[i] : gene_seq_alpha[i] + @inbounds @simd for i in eachindex(chromosome1.genes) + chromosome1.genes[i] = buffer.alpha_operator[i] == 1 ? buffer.child_1_genes[i] : chromosome1.genes[i] end end diff --git a/src/Losses.jl b/src/Losses.jl index 6e9f20e..3a8da0d 100644 --- a/src/Losses.jl +++ b/src/Losses.jl @@ -72,6 +72,7 @@ module LossFunction export get_loss_function using Statistics +using LoopVectorization function floor_to_n10p(x::T) where T<:AbstractFloat abs_x = abs(x) @@ -159,15 +160,48 @@ function r2_score_floor(y_true::AbstractArray{T}, y_pred::AbstractArray{T}) wher return r2_score(y_true_scaled, y_pred_scaled) end -function mean_squared_error(y_true::AbstractArray{T}, y_pred::AbstractArray{T}) where T<:AbstractFloat - d::T = zero(T) - @assert length(y_true) == length(y_pred) - @fastmath @inbounds @simd for i in eachindex(y_true, y_pred) - temp = (y_true[i]-y_pred[i]) - d += temp*temp + +@inline function mean_squared_error_(y_true::AbstractArray{T}, + y_pred::AbstractArray{T}) where {T<:AbstractFloat} + sum = zero(T) + len = length(y_true) + + @fastmath @turbo for i in eachindex(y_true, y_pred) + diff = y_true[i] - y_pred[i] + sum += diff * diff + end + + return sum / len +end + + +@inline function mean_squared_error(y_true::AbstractArray{T}, + y_pred::AbstractArray{T}) where {T<:AbstractFloat} + len = length(y_true) + if len < 100_000 + return mean_squared_error_(y_true, y_pred) + end + + n_chunks = Threads.nthreads() + chunk_size = div(len, n_chunks) + + + partial_sums = zeros(T, n_chunks) + + Threads.@threads for chunk in 1:n_chunks + start_idx = (chunk - 1) * chunk_size + 1 + end_idx = chunk == n_chunks ? len : chunk * chunk_size + + sum = zero(T) + @fastmath @turbo for i in start_idx:end_idx + diff = y_true[i] - y_pred[i] + sum += diff * diff end - return d/length(y_true) -end + partial_sums[chunk] = sum + end + + return sum(partial_sums) / len +end function root_mean_squared_error(y_true::AbstractArray{T}, y_pred::AbstractArray{T}) where T<:AbstractFloat d::T = zero(T) diff --git a/test/Entity_test.jl b/test/Entity_test.jl index 75ec447..5ebc581 100644 --- a/test/Entity_test.jl +++ b/test/Entity_test.jl @@ -85,53 +85,6 @@ end @test typeof(result[1]) <: Real end end - - @testset "Genetic Operators" begin - toolbox = create_test_toolbox() - Random.seed!(42) - chromosome1 = generate_chromosome(toolbox) - chromosome2 = generate_chromosome(toolbox) - - genes1_original = copy(chromosome1.genes) - genes2_original = copy(chromosome2.genes) - - @testset "One Point Crossover" begin - c1, c2 = replicate(chromosome1, chromosome2, toolbox) - gene_one_point_cross_over!(c1, c2) - @test c1.genes != genes1_original || c2.genes != genes2_original - end - - @testset "Mutation" begin - c1 = Chromosome(copy(genes1_original), toolbox) - gene_mutation!(c1, 1.0) # Force mutation - @test c1.genes != genes1_original - end - - @testset "Gene Fusion" begin - c1, c2 = replicate(chromosome1, chromosome2, toolbox) - gene_fussion!(c1, c2, 1.0) # Force fusion - @test c1.genes != genes1_original || c2.genes != genes2_original - end - - @testset "Inversion" begin - pos_11 = toolbox.gene_count - pos_12 = toolbox.gene_count+toolbox.head_len-1 - pos_21 = pos_12 + toolbox.head_len+2 - pos_22 = pos_21 + toolbox.head_len-1 - - c1 = Chromosome(copy(genes1_original), toolbox) - - - initial_head = copy(c1.genes[pos_11:pos_12]) - initial_head2 = copy(c1.genes[pos_21:pos_22]) - - @show initial_head - @show initial_head2 - - gene_inversion!(c1) - @test c1.genes[toolbox.gene_count+1:toolbox.head_len+toolbox.gene_count] != initial_head || initial_head2 != c1.genes[9:11] - end - end @testset "Population Generation" begin toolbox = create_test_toolbox() From 66860862735f26bd71ae4165313213b9407c138d Mon Sep 17 00:00:00 2001 From: maxreiss123 <32074940+maxreiss123@users.noreply.github.com> Date: Tue, 21 Jan 2025 14:59:35 +1100 Subject: [PATCH 28/41] Update README.md --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c1cba8..a809bd2 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,10 @@ The repository contains an implementation of the Gene Expression Programming [1] - Remark: Template for rerunning the test from the paper is located in the paper directory - Remark: the tutorial folder contains notebook, that can be run with google-colab, while showing a step-by-step introduction +# Supported `Engines' for Symbolic Evaluation +- DynamicExpressions.jl +- Flux.jl --> in development + # References - [1] Ferreira, C. (2001). Gene Expression Programming: a New Adaptive Algorithm for Solving Problems. Complex Systems, 13. @@ -106,4 +110,5 @@ The repository contains an implementation of the Gene Expression Programming [1] - [x] Naming conventions! - [x] Improve usability for user interaction - [x] Next operations: Tail flip, Connection symbol flip, staggered exploration -- [ ] Mitigate exception handling in hot paths \ No newline at end of file +- [ ] Mitigate exception handling in hot paths +- [ ] Flexible underlying engine to evaluate the expressions -> Currently DynamicExpressions.jl, Flux in the future for GPU support From 16ea35a4a2360b18f16ca81fa378a5a14a70b282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Thu, 23 Jan 2025 16:44:01 +1100 Subject: [PATCH 29/41] change --- Project.toml | 2 + src/RegressionWrapper.jl | 126 +------------- src/TensorOps.jl | 364 +++++++++++++++++++++++++++++++++++++++ src/Util.jl | 216 +++++++++++++++++++++-- 4 files changed, 570 insertions(+), 138 deletions(-) create mode 100644 src/TensorOps.jl diff --git a/Project.toml b/Project.toml index 580ff2f..b8c9ae4 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" DynamicExpressions = "a40a106e-89c9-4ca8-8020-a735e8728b6b" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" @@ -26,6 +27,7 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Tensors = "48a634ad-e948-5137-8d70-aa71f2a747f4" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index 62aa7d9..be11250 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -27,17 +27,6 @@ simplified interfaces and utilities for symbolic regression tasks with physical - `GENE_COMMON_PROBS`: Default genetic operation probabilities - `FUNCTION_LIB_BACKWARD_COMMON`: Backward dimension propagation functions - `FUNCTION_LIB_FORWARD_COMMON`: Forward dimension propagation functions -- `FUNCTION_LIB_COMMON`: Available mathematical functions - -# Function Library -Includes extensive mathematical operations: -- Basic arithmetic: +, -, *, /, ^ -- Comparisons: min, max -- Rounding: floor, ceil, round -- Exponential: exp, log, log10, log2 -- Trigonometric: sin, cos, tan, asin, acos, atan -- Hyperbolic: sinh, cosh, tanh, asinh, acosh, atanh -- Special: sqr, sqrt, sign, abs # Usage Example ```julia @@ -134,7 +123,7 @@ module RegressionWrapper export GepRegressor export create_function_entries, create_feature_entries, create_constants_entries, create_physical_operations -export GENE_COMMON_PROBS, FUNCTION_LIB_BACKWARD_COMMON, FUNCTION_LIB_FORWARD_COMMON, FUNCTION_LIB_COMMON +export GENE_COMMON_PROBS, FUNCTION_LIB_BACKWARD_COMMON, FUNCTION_LIB_FORWARD_COMMON export fit! export list_all_functions, list_all_arity, list_all_forward_handlers, @@ -160,108 +149,7 @@ using .SBPUtils using .GepUtils using DynamicExpressions using OrderedCollections - -# -#const Toolbox = GepRegression.GepEntities.Toolbox -#const TokenDto = SBPUtils.TokenDto -#const Chromosome = GepRegression.GepEntities.Chromosome -#const EvaluationStrategy = GepRegression.GepEntities.EvaluationStrategy -#const GenericRegressionStrategy = GepRegression.GepEntities.GenericRegressionStrategy -#const StandardRegressionStrategy = GepRegression.GepEntities.StandardRegressionStrategy - -function sqr(x::Vector{T}) where {T<:AbstractFloat} - return x .* x -end - -function sqr(x::T) where {T<:Union{AbstractFloat,Node{<:AbstractFloat}}} - return x * x -end - -""" - FUNCTION_LIB_COMMON::Dict{Symbol,Function} - -Dictionary mapping function symbols to their corresponding functions. -Contains basic mathematical operations, trigonometric, and other common functions. - -# Available Functions -- Basic arithmetic: `+`, `-`, `*`, `/`, `^` -- Comparison: `min`, `max` -- Rounding: `floor`, `ceil`, `round` -- Exponential & Logarithmic: `exp`, `log`, `log10`, `log2` -- Trigonometric: `sin`, `cos`, `tan`, `asin`, `acos`, `atan` -- Hyperbolic: `sinh`, `cosh`, `tanh`, `asinh`, `acosh`, `atanh` -- Other: `abs`, `sqr`, `sqrt`, `sign` - -To add a new function, ensure you also add corresponding entries in `ARITY_LIB_COMMON`, -`FUNCTION_LIB_FORWARD_COMMON`, and `FUNCTION_LIB_BACKWARD_COMMON`. -""" -const FUNCTION_LIB_COMMON = Dict{Symbol,Function}( - :+ => +, - :- => -, - :* => *, - :/ => /, - :^ => ^, - :min => min, - :max => max, :abs => abs, - :floor => floor, - :ceil => ceil, - :round => round, :exp => exp, - :log => log, - :log10 => log10, - :log2 => log2, :sin => sin, - :cos => cos, - :tan => tan, - :asin => asin, - :acos => acos, - :atan => atan, :sinh => sinh, - :cosh => cosh, - :tanh => tanh, - :asinh => asinh, - :acosh => acosh, - :atanh => atanh, :sqr => sqr, - :sqrt => sqrt, :sign => sign -) - -""" - ARITY_LIB_COMMON::Dict{Symbol,Int8} - -Dictionary specifying the number of arguments (arity) for each function in the library. -- Value of 1 indicates unary functions (e.g., `sin`, `cos`, `abs`) -- Value of 2 indicates binary functions (e.g., `+`, `-`, `*`, `/`) - -When adding new functions to `FUNCTION_LIB_COMMON`, ensure to specify their arity here. -""" -const ARITY_LIB_COMMON = Dict{Symbol,Int8}( - :+ => 2, - :- => 2, - :* => 2, - :/ => 2, - :^ => 2, - :min => 2, - :max => 2, - :abs => 1, - :floor => 1, - :ceil => 1, - :round => 1, - :exp => 1, - :log => 1, - :log10 => 1, - :log2 => 1, - :sin => 1, - :cos => 1, - :tan => 1, - :asin => 1, - :acos => 1, - :atan => 1, - :sinh => 1, - :cosh => 1, - :tanh => 1, - :asinh => 1, - :acosh => 1, - :atanh => 1, - :sqrt => 1, - :sqr => 1 -) +using LinearAlgebra """ FUNCTION_LIB_FORWARD_COMMON::Dict{Symbol,Function} @@ -489,7 +377,7 @@ function create_constants_entries( for elem in entered_terminal_nums utilized_symbols[cur_idx] = 0 - nodes[cur_idx] = Node{node_type}(;val=parse(node_type, string(elem))) + nodes[cur_idx] = Node{node_type}(; val=parse(node_type, string(elem))) dimension_information[cur_idx] = get(dimensions_to_consider, elem, ZERO_DIM) cur_idx += 1 end @@ -497,7 +385,7 @@ function create_constants_entries( for _ in 1:rnd_count utilized_symbols[cur_idx] = 0 - nodes[cur_idx] = Node{node_type}(;val=rand()) + nodes[cur_idx] = Node{node_type}(; val=rand()) dimension_information[cur_idx] = ZERO_DIM cur_idx += 1 end @@ -738,7 +626,7 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, x_trai correction_callback=correction_callback, correction_epochs=correction_epochs, correction_amount=correction_amount, - tourni_size=max(Int(ceil(population_size*0.03)),3), + tourni_size=max(Int(ceil(population_size * 0.03)), 3), optimization_epochs=optimization_epochs ) @@ -750,7 +638,7 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, loss_f optimizer_function_::Union{Function,Nothing}=nothing, optimization_epochs::Int=100, hof::Int=3, - correction_epochs::Int=1, + correction_epochs::Int=1, correction_amount::Real=0.05, opt_method_const::Symbol=:cg, target_dimension::Union{Vector{Float16},Nothing}=nothing, @@ -799,7 +687,7 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, loss_f correction_callback=correction_callback, correction_epochs=correction_epochs, correction_amount=correction_amount, - tourni_size=max(Int(ceil(population_size*0.03)),3), + tourni_size=max(Int(ceil(population_size * 0.03)), 3), optimization_epochs=optimization_epochs ) diff --git a/src/TensorOps.jl b/src/TensorOps.jl new file mode 100644 index 0000000..de81646 --- /dev/null +++ b/src/TensorOps.jl @@ -0,0 +1,364 @@ +using Flux, LinearAlgebra, OrderedCollections +using Tensors + +abstract type OperationNode end + +struct InputSelector + idx::Int +end + +@inline function (l::InputSelector)(x::Tuple) + x[l.idx] +end + +struct AdditionNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + AdditionNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct SubtractionNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + SubtractionNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct MultiplicationNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + MultiplicationNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct DivisionNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + DivisionNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct PowerNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + PowerNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct MinNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + MinNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct MaxNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + MaxNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct ContractionNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + ContractionNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct TmagnitudeNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + TmagnitudeNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct InnerProductNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + InnerProductNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct OuterProductNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + OuterProductNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct KroneckerNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + KroneckerNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct UnaryNode <: OperationNode + operation::Function + chain::Union{Nothing,Chain} + use_cuda::Bool + UnaryNode(operation::Function, chain=nothing; use_cuda::Bool=false) = + new(operation, chain, use_cuda) +end + + +for T in subtypes(OperationNode) + @eval Flux.Functors.@functor $T +end + +# Helper for CUDA operations +@inline function ensure_cuda(x, use_cuda::Bool) + #if use_cuda && !isa(x, CuArray) + # cu(x) + #elseif !use_cuda && isa(x, CuArray) + # cpu(x) + # else + return x + # end +end + +@inline function (l::AdditionNode)(x, y) + @show size(x) == size(y) + size(x) == size(y) || throw(DimensionMismatch("Sizes must match exactly. Got sizes $(size(x)) and $(size(y))")) + x, y = ensure_cuda.((x, y), l.use_cuda) + result = x .+ y + l.use_cuda ? result : cpu(result) +end + +@inline function (l::SubtractionNode)(x, y) + size(x) == size(y) || throw(DimensionMismatch("Sizes must match exactly. Got sizes $(size(x)) and $(size(y))")) + x, y = ensure_cuda.((x, y), l.use_cuda) + result = x .- y + l.use_cuda ? result : cpu(result) +end + +@inline function (l::MultiplicationNode)(x::AbstractArray, y::Number) + x = ensure_cuda(x, l.use_cuda) + result = x * y + l.use_cuda ? result : cpu(result) +end + +@inline function (l::MultiplicationNode)(x::AbstractArray, y::AbstractArray) + x, y = ensure_cuda.((x, y), l.use_cuda) + result = x * y + l.use_cuda ? result : cpu(result) +end + +@inline function (l::MultiplicationNode)(x, y) + x, y = ensure_cuda.((x, y), l.use_cuda) + result = x * y + l.use_cuda ? result : cpu(result) +end + +@inline function (l::DivisionNode)(x::Number, y::AbstractArray) + y = ensure_cuda(y, l.use_cuda) + result = x / y + l.use_cuda ? result : cpu(result) +end + +@inline function (l::DivisionNode)(x::AbstractArray, y::Number) + x = ensure_cuda(x, l.use_cuda) + result = x / y + l.use_cuda ? result : cpu(result) +end + +@inline function (l::PowerNode)(x, y) + x, y = ensure_cuda.((x, y), l.use_cuda) + result = x .^ y + l.use_cuda ? result : cpu(result) +end + +@inline function (l::MinNode)(x, y) + x, y = ensure_cuda.((x, y), l.use_cuda) + result = min.(x, y) + l.use_cuda ? result : cpu(result) +end + +@inline function (l::MaxNode)(x, y) + x, y = ensure_cuda.((x, y), l.use_cuda) + result = max.(x, y) + l.use_cuda ? result : cpu(result) +end + +@inline function (l::UnaryNode)(x) + x = ensure_cuda(x, l.use_cuda) + result = l.operation.(x) + l.use_cuda ? result : cpu(result) +end + +@inline function (l::ContractionNode)(x) + x = ensure_cuda(x, l.use_cuda) + result = sum(x, dims=1:(ndims(x)-1)) + l.use_cuda ? result : cpu(result) +end + +@inline function (l::TmagnitudeNode)(x) + x = ensure_cuda(x, l.use_cuda) + result = sqrt.(sum(abs2.(x), dims=1:(ndims(x)-1))) + l.use_cuda ? result : cpu(result) +end + +@inline function (l::InnerProductNode)(x, y) + x, y = ensure_cuda.((x, y), l.use_cuda) + result = sum(x .* y, dims=1:(ndims(x)-1)) + l.use_cuda ? result : cpu(result) +end + +@inline function (l::OuterProductNode)(x, y) + x, y = ensure_cuda.((x, y), l.use_cuda) + x_reshape = reshape(x, size(x)..., 1, :) + y_reshape = reshape(y, 1, size(y)...) + result = x_reshape .* y_reshape + l.use_cuda ? result : cpu(result) +end + +@inline function (l::KroneckerNode)(x) + x = ensure_cuda(x, l.use_cuda) + batch_size = size(x)[end] + I_mat = Matrix(I, 3, 3) + I_mat = l.use_cuda ? cu(I_mat) : I_mat + result = similar(x, size(x, 1) * 3, size(x, 2) * 3, batch_size) + result = l.use_cuda ? cu(result) : result + + for b in 1:batch_size + result[:, :, b] = kron(x[:, :, b], I_mat) + end + return result +end + +# Operation mappings +const UNARY_OPS = Dict( + :exp => exp, :log => log, :sqrt => sqrt, + :sin => sin, :cos => cos, :tan => tan, + :asin => asin, :acos => acos, :atan => atan, + :sinh => sinh, :cosh => cosh, :tanh => tanh, + :asinh => asinh, :acosh => acosh, :atanh => atanh, + :abs => abs, :sign => sign, + :floor => floor, :ceil => ceil, :round => round +) + +const BINARY_OPS = Dict( + :+ => AdditionNode, + :- => SubtractionNode, + :* => MultiplicationNode, + :/ => DivisionNode, + :^ => PowerNode, + :min => MinNode, + :max => MaxNode +) + +const TENSOR_OPS = Dict( + :contract => ContractionNode, + :magnitude => TmagnitudeNode, + :kronecker => KroneckerNode, + :inner => InnerProductNode, + :outer => OuterProductNode +) + +# Optimized network compilation +function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict; use_cuda::Bool=false) + stack = Vector{Any}(undef, length(rek_string)) + sp = 0 + input_count = count(x -> x isa Int8, rek_string) + inputs = [InputSelector(i) for i in 1:input_count] + + @inbounds for elem in rek_string + if elem isa Int8 + sp += 1 + stack[sp] = inputs[elem] + else + arity = get(arity_map, elem, 0) + if arity == 2 && sp >= 2 + op1 = stack[sp] + op2 = stack[sp-1] + sp -= 1 + + if haskey(BINARY_OPS, elem) + node = BINARY_OPS[elem](nothing, use_cuda=use_cuda) + stack[sp] = x -> node(op2(x), op1(x)) + elseif haskey(TENSOR_OPS, elem) + node = TENSOR_OPS[elem](nothing, use_cuda=use_cuda) + stack[sp] = x -> node(op2(x), op1(x)) + end + elseif arity == 1 && sp >= 1 + op = stack[sp] + if haskey(UNARY_OPS, elem) + node = UnaryNode(UNARY_OPS[elem], nothing, use_cuda=use_cuda) + stack[sp] = x -> node(op(x)) + elseif haskey(TENSOR_OPS, elem) + node = TENSOR_OPS[elem](nothing, use_cuda=use_cuda) + stack[sp] = x -> node(op(x)) + end + end + end + end + + @assert sp == 1 "Invalid expression: stack error" + return Chain(stack[1]) +end + +# Test functions +function test_binary_ops(batch_size::Int=32) + # Test addition with batching + add_node = AdditionNode() + x = rand(Float32, 10, batch_size) # (features, batch) + y = rand(Float32, 10, batch_size) + result = add_node(x, y) + @assert size(result, 2) == batch_size "Batch dimension not preserved" + + # Test multiplication with batching + mul_node = MultiplicationNode() + result = mul_node(x, y) + @assert size(result, 2) == batch_size "Batch dimension not preserved" + + return (addition=result, multiplication=result) +end + +function test_tensor_ops(batch_size::Int=32) + # Test inner product with batching + inner_node = InnerProductNode() + x = rand(Float32, 10, batch_size) + y = rand(Float32, 10, batch_size) + result = inner_node(x, y) + @assert size(result)[end] == batch_size "Batch dimension not preserved" + + # Test contraction with batching + contract_node = ContractionNode() + tensor = rand(Float32, 5, 5, batch_size) + result = contract_node(tensor) + @assert size(result)[end] == batch_size "Batch dimension not preserved" + + return (inner_product=result, contraction=result) +end + +function test_complete_network(batch_size::Int=32, use_cuda::Bool=false) + rek_string = [Int8(1), Int8(2), :*, Int8(3), :+] + + arity_map = OrderedDict{Any,Int}( + :+ => 2, + :* => 2 + ) + + network = compile_to_flux_network(rek_string, arity_map, use_cuda=use_cuda) + + x1 = rand(Float32, 100, batch_size) + x2 = rand(Float32, 100, batch_size) + x3 = rand(Float32, 100, batch_size) + + if use_cuda + x1, x2, x3 = cu.((x1, x2, x3)) + end + + result = network((x1, x2, x3)) + @assert size(result)[end] == batch_size "Batch dimension not preserved in network" + + return network, result +end + +# Run tests +function run_all_tests(batch_size::Int=32) + println("Testing binary operations...") + binary_results = test_binary_ops(batch_size) + + println("Testing tensor operations...") + tensor_results = test_tensor_ops(batch_size) + + println("Testing complete network...") + network, complete_results = test_complete_network(batch_size) + + println("All tests completed successfully!") + return (binary=binary_results, tensor=tensor_results, network=complete_results) +end + +# Run all tests +results = run_all_tests(10000000) \ No newline at end of file diff --git a/src/Util.jl b/src/Util.jl index 31fe4b0..2ace933 100644 --- a/src/Util.jl +++ b/src/Util.jl @@ -48,6 +48,20 @@ operations, including optimization, history tracking, data manipulation, and sta - `train_test_split`: Split dataset for training/testing - `save_state`, `load_state`: State persistence +## Constants +- `FUNCTION_LIB_COMMON`: Available mathematical functions + +# Function Library +Includes extensive mathematical operations: +- Basic arithmetic: +, -, *, /, ^ +- Comparisons: min, max +- Rounding: floor, ceil, round +- Exponential: exp, log, log10, log2 +- Trigonometric: sin, cos, tan, asin, acos, atan +- Hyperbolic: sinh, cosh, tanh, asinh, acosh, atanh +- Special: sqr, sqrt, sign, abs +- Tensorfunctions: + # Usage Example ```julia # History recording @@ -86,7 +100,10 @@ optimized_node, final_loss = optimize_constants!( - `Zygote`: Automatic differentiation - `Serialization`: State persistence - `Statistics`: Statistical computations +- `Flux`: ML-Package +- `Tensors`: mathematical objects of higher order - `Random`: Random number generation +- `CUDA`: Extension to run on CUDA-cores """ module GepUtils @@ -95,6 +112,8 @@ export save_state, load_state export create_history_recorder, record_history!, record!, close_recorder! export HistoryRecorder, OptimizationHistory, get_history_arrays export train_test_split +export FUNCTION_LIB_COMMON, ARITY_LIB_COMMON +export TensorNode, compile_network using OrderedCollections using DynamicExpressions @@ -105,10 +124,122 @@ using Zygote using Serialization using Statistics using Random +using Tensors +using Flux using Base.Threads: @spawn +function sqr(x::Vector{T}) where {T<:AbstractFloat} + return x .* x +end + +function sqr(x::T) where {T<:Union{AbstractFloat,Node{<:AbstractFloat}}} + return x * x +end + +""" + FUNCTION_LIB_COMMON::Dict{Symbol,Function} + +Dictionary mapping function symbols to their corresponding functions. +Contains basic mathematical operations, trigonometric, and other common functions. + +# Available Functions +- Basic arithmetic: `+`, `-`, `*`, `/`, `^` +- Comparison: `min`, `max` +- Rounding: `floor`, `ceil`, `round` +- Exponential & Logarithmic: `exp`, `log`, `log10`, `log2` +- Trigonometric: `sin`, `cos`, `tan`, `asin`, `acos`, `atan` +- Hyperbolic: `sinh`, `cosh`, `tanh`, `asinh`, `acosh`, `atanh` +- Other: `abs`, `sqr`, `sqrt`, `sign` +- Tensor Functions: + +To add a new function, ensure you also add corresponding entries in `ARITY_LIB_COMMON`, +`FUNCTION_LIB_FORWARD_COMMON`, and `FUNCTION_LIB_BACKWARD_COMMON`. +""" +const FUNCTION_LIB_COMMON = Dict{Symbol,Function}( + :+ => +, + :- => -, + :* => *, + :/ => /, + :^ => ^, + :min => min, + :max => max, :abs => abs, + :floor => floor, + :ceil => ceil, + :round => round, :exp => exp, + :log => log, + :log10 => log10, + :log2 => log2, :sin => sin, + :cos => cos, + :tan => tan, + :asin => asin, + :acos => acos, + :atan => atan, :sinh => sinh, + :cosh => cosh, + :tanh => tanh, + :asinh => asinh, + :acosh => acosh, + :atanh => atanh, :sqr => sqr, + :sqrt => sqrt, :sign => sign, + + + # Tensor specific operations -> never use in scalar regression tasks - documenation: https://ferrite-fem.github.io/Tensors.jl/stable/man/other_operators/ + :inv => inv, :tr => tr, :det => det, + :symmetric => symmetric, :skew => skew, + :vol => vol, :dev => dev, + :dot => dot, :dcontract => dcontract, + :tdot => tdot, :dott => dott +) + +""" + ARITY_LIB_COMMON::Dict{Symbol,Int8} + +Dictionary specifying the number of arguments (arity) for each function in the library. +- Value of 1 indicates unary functions (e.g., `sin`, `cos`, `abs`) +- Value of 2 indicates binary functions (e.g., `+`, `-`, `*`, `/`) + +When adding new functions to `FUNCTION_LIB_COMMON`, ensure to specify their arity here. +""" +const ARITY_LIB_COMMON = Dict{Symbol,Int8}( + :+ => 2, + :- => 2, + :* => 2, + :/ => 2, + :^ => 2, + :min => 2, + :max => 2, + :abs => 1, + :floor => 1, + :ceil => 1, + :round => 1, + :exp => 1, + :log => 1, + :log10 => 1, + :log2 => 1, + :sin => 1, + :cos => 1, + :tan => 1, + :asin => 1, + :acos => 1, + :atan => 1, + :sinh => 1, + :cosh => 1, + :tanh => 1, + :asinh => 1, + :acosh => 1, + :atanh => 1, + :sqrt => 1, + :sqr => 1, + + # Tensor specific operations -> never use in scalar regression tasks - documenation: https://ferrite-fem.github.io/Tensors.jl/stable/man/other_operators/ + :inv => 1, :tr => 1, :det => 1, + :symmetric => 1, :skew => 1, + :vol => 1, :dev => 1, + :dot => 2, :dcontract => 2, + :tdot => 1, :dott => 1 +) + struct OptimizationHistory{T<:Union{AbstractFloat,Tuple}} train_loss::Vector{T} val_loss::Vector{T} @@ -258,12 +389,12 @@ end @inline function tuple_agg(entries::Vector{T}, fun::Function) where {T<:Tuple} - isempty(entries) && return entries[1] + isempty(entries) && return entries[1] N = length(first(entries)) L = length(entries) - + vectors = ntuple(i -> Vector{Float64}(undef, L), N) - + for (j, entry) in enumerate(entries) for i in 1:length(entry) vectors[i][j] = entry[i] @@ -278,10 +409,10 @@ end ) where {T<:Union{AbstractFloat,Tuple}} for (epoch, train_loss, val_loss, fit_vector) in channel @inbounds begin - history.train_loss[epoch] = train_loss - history.val_loss[epoch] = val_loss - #history.train_mean[epoch] = tuple_agg(fit_vector,mean) - #history.train_std[epoch] = tuple_agg(fit_vector, std) + history.train_loss[epoch] = train_loss + history.val_loss[epoch] = val_loss + #history.train_mean[epoch] = tuple_agg(fit_vector,mean) + #history.train_std[epoch] = tuple_agg(fit_vector, std) end end end @@ -435,7 +566,7 @@ function compile_djl_datatype(rek_string::Vector, arity_map::OrderedDict, callba push!(stack, elem isa Int8 ? nodes[elem] : elem) end end - return pre_len==1 ? last(stack) : stack + return pre_len == 1 ? last(stack) : stack end @inline function retrieve_constants_from_node(node::Node) @@ -448,6 +579,53 @@ end constants end + +""" +Compile expression as Flux network +""" + +function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict; use_cuda::Bool=false) + stack = Vector{Any}(undef, length(rek_string)) + sp = 0 + input_count = count(x -> x isa Int8, rek_string) + inputs = [InputSelector(i) for i in 1:input_count] + + @inbounds for elem in rek_string + if elem isa Int8 + sp += 1 + stack[sp] = inputs[elem] + else + arity = get(arity_map, elem, 0) + if arity == 2 && sp >= 2 + op1 = stack[sp] + op2 = stack[sp-1] + sp -= 1 + + if haskey(BINARY_OPS, elem) + node = BINARY_OPS[elem](nothing, use_cuda=use_cuda) + stack[sp] = x -> node(op2(x), op1(x)) + elseif haskey(TENSOR_OPS, elem) + node = TENSOR_OPS[elem](nothing, use_cuda=use_cuda) + stack[sp] = x -> node(op2(x), op1(x)) + end + elseif arity == 1 && sp >= 1 + op = stack[sp] + if haskey(UNARY_OPS, elem) + node = UnaryNode(UNARY_OPS[elem], nothing, use_cuda=use_cuda) + stack[sp] = x -> node(op(x)) + elseif haskey(TENSOR_OPS, elem) + node = TENSOR_OPS[elem](nothing, use_cuda=use_cuda) + stack[sp] = x -> node(op(x)) + end + end + end + end + + @assert sp == 1 "Invalid expression: stack error" + return Chain(stack[1]) +end + + """ optimize_constants!( node::Node, @@ -624,32 +802,32 @@ end function train_test_split( - X::AbstractMatrix{T}, - y::AbstractVector{T}; - train_ratio::T=0.9, + X::AbstractMatrix{T}, + y::AbstractVector{T}; + train_ratio::T=0.9, consider::Int=1 ) where {T<:AbstractFloat} data = hcat(X, y) - + data = data[shuffle(1:size(data, 1)), :] - + split_point = floor(Int, size(data, 1) * train_ratio) - + data_train = data[1:split_point, :] - data_test = data[(split_point + 1):end, :] - + data_test = data[(split_point+1):end, :] + x_train = T.(data_train[1:consider:end, 1:(end-1)]) y_train = T.(data_train[1:consider:end, end]) - + x_test = T.(data_test[1:consider:end, 1:(end-1)]) y_test = T.(data_test[1:consider:end, end]) - - return x_train, y_train, x_test, y_test + + return x_train, y_train, x_test, y_test end From 2cd7ccd9a0ae1364fae8fdfd5a9b0bd44a4c4803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Fri, 24 Jan 2025 13:36:19 +1100 Subject: [PATCH 30/41] hide cuda options --- Project.toml | 1 + src/TensorOps.jl | 508 ++++++++++++++++++++++++----------------------- src/Util.jl | 46 ----- 3 files changed, 265 insertions(+), 290 deletions(-) diff --git a/Project.toml b/Project.toml index b8c9ae4..2980c65 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.3.4" [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" diff --git a/src/TensorOps.jl b/src/TensorOps.jl index de81646..ca4df9a 100644 --- a/src/TensorOps.jl +++ b/src/TensorOps.jl @@ -1,364 +1,384 @@ -using Flux, LinearAlgebra, OrderedCollections +module TensorRegUtils + +using Flux, LinearAlgebra, OrderedCollections, ChainRulesCore + using Tensors +# Types +export OperationNode +export InputSelector +export AdditionNode, SubtractionNode, MultiplicationNode, DivisionNode, PowerNode +export MinNode, MaxNode, InversionNode +export TraceNode, DeterminantNode, SymmetricNode, SkewNode +export VolumetricNode, DeviatricNode, TdotNode, DottNode +export DoubleContractionNode, DeviatoricNode +export ConstantNode, UnaryNode + +# Core functions +export compile_to_flux_network + abstract type OperationNode end struct InputSelector idx::Int end -@inline function (l::InputSelector)(x::Tuple) +@inline function (l::InputSelector)(x::Tuple) x[l.idx] end struct AdditionNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - AdditionNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + AdditionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end struct SubtractionNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - SubtractionNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + SubtractionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end struct MultiplicationNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - MultiplicationNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + MultiplicationNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end struct DivisionNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - DivisionNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + DivisionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end struct PowerNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - PowerNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + PowerNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end struct MinNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - MinNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + MinNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end struct MaxNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - MaxNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + MaxNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct InversionNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + InversionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct TraceNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + TraceNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct DeterminantNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + DeterminantNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct SymmetricNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + SymmetricNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end -struct ContractionNode <: OperationNode +struct SkewNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - ContractionNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + SkewNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end -struct TmagnitudeNode <: OperationNode +struct VolumetricNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - TmagnitudeNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + VolumetricNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end -struct InnerProductNode <: OperationNode +struct DeviatricNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - InnerProductNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + DeviatricNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end -struct OuterProductNode <: OperationNode +struct TdotNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - OuterProductNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + TdotNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end -struct KroneckerNode <: OperationNode +struct DottNode <: OperationNode chain::Union{Nothing,Chain} use_cuda::Bool - KroneckerNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) + DottNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) end +struct DoubleContractionNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + DoubleContractionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct DeviatoricNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + DeviatoricNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +end + +struct ConstantNode <: OperationNode + value::Number + chain::Union{Nothing,Chain} + use_cuda::Bool + ConstantNode(value::Number, chain=Chain(); use_cuda::Bool=false) = new(value, chain, use_cuda) +end + + struct UnaryNode <: OperationNode operation::Function chain::Union{Nothing,Chain} use_cuda::Bool - UnaryNode(operation::Function, chain=nothing; use_cuda::Bool=false) = + UnaryNode(operation::Function, chain=Chain(); use_cuda::Bool=false) = new(operation, chain, use_cuda) end -for T in subtypes(OperationNode) - @eval Flux.Functors.@functor $T +const NODES = [ + AdditionNode, + SubtractionNode, + MultiplicationNode, + DivisionNode, + PowerNode, + MinNode, + MaxNode, + InversionNode, + TraceNode, + DeterminantNode, + SymmetricNode, + SkewNode, + VolumetricNode, + DeviatricNode, + TdotNode, + DottNode, + DoubleContractionNode, + DeviatoricNode, + ConstantNode, + UnaryNode +] + +for T in NODES + @eval Flux.Functors.@functor $T +end + +@inline function (l::AdditionNode)(x::Tensor, y::Tensor) + result = x .+ y + return result end -# Helper for CUDA operations -@inline function ensure_cuda(x, use_cuda::Bool) - #if use_cuda && !isa(x, CuArray) - # cu(x) - #elseif !use_cuda && isa(x, CuArray) - # cpu(x) - # else - return x - # end +@inline function (l::AdditionNode)(x::Vector, y::Vector) + result = x .+ y + return result end -@inline function (l::AdditionNode)(x, y) - @show size(x) == size(y) - size(x) == size(y) || throw(DimensionMismatch("Sizes must match exactly. Got sizes $(size(x)) and $(size(y))")) - x, y = ensure_cuda.((x, y), l.use_cuda) - result = x .+ y - l.use_cuda ? result : cpu(result) +@inline function (l::AdditionNode)(x::Number, y::Number) + result = x .+ y + return result end -@inline function (l::SubtractionNode)(x, y) - size(x) == size(y) || throw(DimensionMismatch("Sizes must match exactly. Got sizes $(size(x)) and $(size(y))")) - x, y = ensure_cuda.((x, y), l.use_cuda) - result = x .- y - l.use_cuda ? result : cpu(result) +@inline function (l::SubtractionNode)(x::Tensor, y::Tensor) + result = x .- y + return result +end + + +@inline function (l::SubtractionNode)(x::Vector, y::Vector) + result = x .- y + return result +end + +@inline function (l::SubtractionNode)(x::Number, y::Number) + result = x .- y + return result +end + +@inline function (l::MultiplicationNode)(x::Number, y::Number) + result = y .* x + return result +end + +@inline function (l::MultiplicationNode)(x::Tensor, y::Number) + result = y * x + return result end -@inline function (l::MultiplicationNode)(x::AbstractArray, y::Number) - x = ensure_cuda(x, l.use_cuda) - result = x * y - l.use_cuda ? result : cpu(result) +@inline function (l::MultiplicationNode)(x::Number, y::Tensor) + result = y * x + return result end -@inline function (l::MultiplicationNode)(x::AbstractArray, y::AbstractArray) - x, y = ensure_cuda.((x, y), l.use_cuda) - result = x * y - l.use_cuda ? result : cpu(result) +@inline function (l::MultiplicationNode)(x::Tensor, y::Tensor) + result = dot(x,y) + return result end -@inline function (l::MultiplicationNode)(x, y) - x, y = ensure_cuda.((x, y), l.use_cuda) - result = x * y - l.use_cuda ? result : cpu(result) +@inline function (l::DivisionNode)(x::Tensor, y::Vector) + result = x ./ y + return result end -@inline function (l::DivisionNode)(x::Number, y::AbstractArray) - y = ensure_cuda(y, l.use_cuda) - result = x / y - l.use_cuda ? result : cpu(result) + +@inline function (l::PowerNode)(x::Union{Tensor,Number}, y::Number) + result = x ^ y + return result end -@inline function (l::DivisionNode)(x::AbstractArray, y::Number) - x = ensure_cuda(x, l.use_cuda) - result = x / y - l.use_cuda ? result : cpu(result) +@inline function (l::DoubleContractionNode)(x, y) + result = dcontract(x, y) + return result end -@inline function (l::PowerNode)(x, y) - x, y = ensure_cuda.((x, y), l.use_cuda) - result = x .^ y - l.use_cuda ? result : cpu(result) +@inline function (l::DeviatoricNode)(x) + result = dev(x) + return result end + @inline function (l::MinNode)(x, y) - x, y = ensure_cuda.((x, y), l.use_cuda) result = min.(x, y) - l.use_cuda ? result : cpu(result) + return result end @inline function (l::MaxNode)(x, y) - x, y = ensure_cuda.((x, y), l.use_cuda) result = max.(x, y) - l.use_cuda ? result : cpu(result) + return result end -@inline function (l::UnaryNode)(x) - x = ensure_cuda(x, l.use_cuda) - result = l.operation.(x) - l.use_cuda ? result : cpu(result) +@inline function (l::InversionNode)(x) + result = inv(x) + return result end -@inline function (l::ContractionNode)(x) - x = ensure_cuda(x, l.use_cuda) - result = sum(x, dims=1:(ndims(x)-1)) - l.use_cuda ? result : cpu(result) +@inline function (l::TraceNode)(x) + result = tr(x) + return result end -@inline function (l::TmagnitudeNode)(x) - x = ensure_cuda(x, l.use_cuda) - result = sqrt.(sum(abs2.(x), dims=1:(ndims(x)-1))) - l.use_cuda ? result : cpu(result) +@inline function (l::DeterminantNode)(x) + result = det(x) + return result end -@inline function (l::InnerProductNode)(x, y) - x, y = ensure_cuda.((x, y), l.use_cuda) - result = sum(x .* y, dims=1:(ndims(x)-1)) - l.use_cuda ? result : cpu(result) +@inline function (l::SymmetricNode)(x) + result = symmetric(x) + return result end -@inline function (l::OuterProductNode)(x, y) - x, y = ensure_cuda.((x, y), l.use_cuda) - x_reshape = reshape(x, size(x)..., 1, :) - y_reshape = reshape(y, 1, size(y)...) - result = x_reshape .* y_reshape - l.use_cuda ? result : cpu(result) +@inline function (l::SkewNode)(x) + result = skew(x) + return result end -@inline function (l::KroneckerNode)(x) - x = ensure_cuda(x, l.use_cuda) - batch_size = size(x)[end] - I_mat = Matrix(I, 3, 3) - I_mat = l.use_cuda ? cu(I_mat) : I_mat - result = similar(x, size(x, 1) * 3, size(x, 2) * 3, batch_size) - result = l.use_cuda ? cu(result) : result - - for b in 1:batch_size - result[:, :, b] = kron(x[:, :, b], I_mat) - end +@inline function (l::VolumetricNode)(x) + result = vol(x) return result end -# Operation mappings -const UNARY_OPS = Dict( - :exp => exp, :log => log, :sqrt => sqrt, - :sin => sin, :cos => cos, :tan => tan, - :asin => asin, :acos => acos, :atan => atan, - :sinh => sinh, :cosh => cosh, :tanh => tanh, - :asinh => asinh, :acosh => acosh, :atanh => atanh, - :abs => abs, :sign => sign, - :floor => floor, :ceil => ceil, :round => round -) - -const BINARY_OPS = Dict( - :+ => AdditionNode, - :- => SubtractionNode, - :* => MultiplicationNode, - :/ => DivisionNode, - :^ => PowerNode, - :min => MinNode, - :max => MaxNode -) - -const TENSOR_OPS = Dict( - :contract => ContractionNode, - :magnitude => TmagnitudeNode, - :kronecker => KroneckerNode, - :inner => InnerProductNode, - :outer => OuterProductNode -) - -# Optimized network compilation -function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict; use_cuda::Bool=false) - stack = Vector{Any}(undef, length(rek_string)) - sp = 0 - input_count = count(x -> x isa Int8, rek_string) - inputs = [InputSelector(i) for i in 1:input_count] - - @inbounds for elem in rek_string - if elem isa Int8 - sp += 1 - stack[sp] = inputs[elem] +@inline function (l::DeviatricNode)(x) + result = dev(x) + return result +end + +@inline function (l::TdotNode)(x) + result = tdot(x) + return result +end + +@inline function (l::DottNode)(x) + result = dott(x) + return result +end + +@inline function (l::ConstantNode)(x) + return l.value +end + + +@inline function (l::UnaryNode)(x) + result = l.operation.(x) + return result +end + +function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict; use_cuda::Bool=false) + stack = [] + inputs_idx = Dict{Int8, Int8}() + idx = 1 + for elem in reverse(rek_string) + if get(arity_map, elem, 0) == 2 + op1 = pop!(stack) + op2 = pop!(stack) + node = callbacks[elem]() + push!(stack, (inputs) -> node(op1(inputs), op2(inputs))) + @show elem + elseif get(arity_map, elem, 0) == 1 + op = pop!(stack) + node = callbacks[elem]() + push!(stack, (inputs) -> node(op(inputs))) else - arity = get(arity_map, elem, 0) - if arity == 2 && sp >= 2 - op1 = stack[sp] - op2 = stack[sp-1] - sp -= 1 - - if haskey(BINARY_OPS, elem) - node = BINARY_OPS[elem](nothing, use_cuda=use_cuda) - stack[sp] = x -> node(op2(x), op1(x)) - elseif haskey(TENSOR_OPS, elem) - node = TENSOR_OPS[elem](nothing, use_cuda=use_cuda) - stack[sp] = x -> node(op2(x), op1(x)) - end - elseif arity == 1 && sp >= 1 - op = stack[sp] - if haskey(UNARY_OPS, elem) - node = UnaryNode(UNARY_OPS[elem], nothing, use_cuda=use_cuda) - stack[sp] = x -> node(op(x)) - elseif haskey(TENSOR_OPS, elem) - node = TENSOR_OPS[elem](nothing, use_cuda=use_cuda) - stack[sp] = x -> node(op(x)) - end + if get(inputs_idx, elem, 0) == 0 + inputs_idx[elem] = idx + idx +=1 + end + if nodes[elem] isa Number + push!(stack, _ -> nodes[elem]) + #push!(stack, (inputs) -> ConstantNode(inputs_idx[elem])(inputs)) + else + push!(stack, (inputs) -> InputSelector(inputs_idx[elem])(inputs)) end end + end - - @assert sp == 1 "Invalid expression: stack error" - return Chain(stack[1]) -end - -# Test functions -function test_binary_ops(batch_size::Int=32) - # Test addition with batching - add_node = AdditionNode() - x = rand(Float32, 10, batch_size) # (features, batch) - y = rand(Float32, 10, batch_size) - result = add_node(x, y) - @assert size(result, 2) == batch_size "Batch dimension not preserved" - - # Test multiplication with batching - mul_node = MultiplicationNode() - result = mul_node(x, y) - @assert size(result, 2) == batch_size "Batch dimension not preserved" - - return (addition=result, multiplication=result) -end - -function test_tensor_ops(batch_size::Int=32) - # Test inner product with batching - inner_node = InnerProductNode() - x = rand(Float32, 10, batch_size) - y = rand(Float32, 10, batch_size) - result = inner_node(x, y) - @assert size(result)[end] == batch_size "Batch dimension not preserved" - - # Test contraction with batching - contract_node = ContractionNode() - tensor = rand(Float32, 5, 5, batch_size) - result = contract_node(tensor) - @assert size(result)[end] == batch_size "Batch dimension not preserved" - - return (inner_product=result, contraction=result) -end - -function test_complete_network(batch_size::Int=32, use_cuda::Bool=false) - rek_string = [Int8(1), Int8(2), :*, Int8(3), :+] - - arity_map = OrderedDict{Any,Int}( - :+ => 2, - :* => 2 - ) - - network = compile_to_flux_network(rek_string, arity_map, use_cuda=use_cuda) - - x1 = rand(Float32, 100, batch_size) - x2 = rand(Float32, 100, batch_size) - x3 = rand(Float32, 100, batch_size) - - if use_cuda - x1, x2, x3 = cu.((x1, x2, x3)) + return Chain(pop!(stack)), inputs_idx +end + +""" + +Later on cuda Module + +@inline function ensure_cuda(x, use_cuda::Bool) + if use_cuda && !isa(x, CuArray) + cu(x) + elseif !use_cuda && isa(x, CuArray) + cpu(x) + else + return x end - - result = network((x1, x2, x3)) - @assert size(result)[end] == batch_size "Batch dimension not preserved in network" - - return network, result -end - -# Run tests -function run_all_tests(batch_size::Int=32) - println("Testing binary operations...") - binary_results = test_binary_ops(batch_size) - - println("Testing tensor operations...") - tensor_results = test_tensor_ops(batch_size) - - println("Testing complete network...") - network, complete_results = test_complete_network(batch_size) - - println("All tests completed successfully!") - return (binary=binary_results, tensor=tensor_results, network=complete_results) -end - -# Run all tests -results = run_all_tests(10000000) \ No newline at end of file +end + +Example Method usage: + +struct DottNode <: OperationNode + chain::Union{Nothing,Chain} + use_cuda::Bool + DottNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) +end + +notes: +if rek comes to a constant: ConstantNode(elem, nothing, use_cuda=use_cuda) +if rek comes to an x -> InputSelector(i) +if unary -> initializes Arity1Node(nothing, use_cuda=use_cuda) +if binary -> initaliszed Arity2Node(nothing, use_cuda=use_cuda) + +@assert sp == 1 "Invalid expression: stack error" + +""" + +end \ No newline at end of file diff --git a/src/Util.jl b/src/Util.jl index 2ace933..0dcfe97 100644 --- a/src/Util.jl +++ b/src/Util.jl @@ -580,52 +580,6 @@ end end -""" -Compile expression as Flux network -""" - -function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict; use_cuda::Bool=false) - stack = Vector{Any}(undef, length(rek_string)) - sp = 0 - input_count = count(x -> x isa Int8, rek_string) - inputs = [InputSelector(i) for i in 1:input_count] - - @inbounds for elem in rek_string - if elem isa Int8 - sp += 1 - stack[sp] = inputs[elem] - else - arity = get(arity_map, elem, 0) - if arity == 2 && sp >= 2 - op1 = stack[sp] - op2 = stack[sp-1] - sp -= 1 - - if haskey(BINARY_OPS, elem) - node = BINARY_OPS[elem](nothing, use_cuda=use_cuda) - stack[sp] = x -> node(op2(x), op1(x)) - elseif haskey(TENSOR_OPS, elem) - node = TENSOR_OPS[elem](nothing, use_cuda=use_cuda) - stack[sp] = x -> node(op2(x), op1(x)) - end - elseif arity == 1 && sp >= 1 - op = stack[sp] - if haskey(UNARY_OPS, elem) - node = UnaryNode(UNARY_OPS[elem], nothing, use_cuda=use_cuda) - stack[sp] = x -> node(op(x)) - elseif haskey(TENSOR_OPS, elem) - node = TENSOR_OPS[elem](nothing, use_cuda=use_cuda) - stack[sp] = x -> node(op(x)) - end - end - end - end - - @assert sp == 1 "Invalid expression: stack error" - return Chain(stack[1]) -end - - """ optimize_constants!( node::Node, From 0dcc9c3871497b50e4b3342e1c94529b826bedcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Fri, 24 Jan 2025 15:25:55 +1100 Subject: [PATCH 31/41] green tests for network --- src/TensorOps.jl | 36 +++--- test/runtests.jl | 3 +- test/tensor_ops_test.jl | 250 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 269 insertions(+), 20 deletions(-) create mode 100644 test/tensor_ops_test.jl diff --git a/src/TensorOps.jl b/src/TensorOps.jl index ca4df9a..743085f 100644 --- a/src/TensorOps.jl +++ b/src/TensorOps.jl @@ -179,64 +179,64 @@ for T in NODES @eval Flux.Functors.@functor $T end -@inline function (l::AdditionNode)(x::Tensor, y::Tensor) - result = x .+ y +@inline function (l::AdditionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) + result = x + y return result end @inline function (l::AdditionNode)(x::Vector, y::Vector) - result = x .+ y + result = x + y return result end @inline function (l::AdditionNode)(x::Number, y::Number) - result = x .+ y + result = x + y return result end -@inline function (l::SubtractionNode)(x::Tensor, y::Tensor) - result = x .- y +@inline function (l::SubtractionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) + result = x - y return result end @inline function (l::SubtractionNode)(x::Vector, y::Vector) - result = x .- y + result = x - y return result end @inline function (l::SubtractionNode)(x::Number, y::Number) - result = x .- y + result = x - y return result end @inline function (l::MultiplicationNode)(x::Number, y::Number) - result = y .* x + result = y * x return result end -@inline function (l::MultiplicationNode)(x::Tensor, y::Number) +@inline function (l::MultiplicationNode)(x::Union{Tensor,SymmetricTensor}, y::Number) result = y * x return result end -@inline function (l::MultiplicationNode)(x::Number, y::Tensor) +@inline function (l::MultiplicationNode)(x::Number, y::Union{Tensor,SymmetricTensor}) result = y * x return result end -@inline function (l::MultiplicationNode)(x::Tensor, y::Tensor) +@inline function (l::MultiplicationNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) result = dot(x,y) return result end -@inline function (l::DivisionNode)(x::Tensor, y::Vector) - result = x ./ y +@inline function (l::DivisionNode)(x::Union{Tensor,SymmetricTensor}, y::Vector) + result = x / y return result end -@inline function (l::PowerNode)(x::Union{Tensor,Number}, y::Number) +@inline function (l::PowerNode)(x::Union{Tensor,SymmetricTensor,Number}, y::Number) result = x ^ y return result end @@ -253,12 +253,12 @@ end @inline function (l::MinNode)(x, y) - result = min.(x, y) + result = min(x, y) return result end @inline function (l::MaxNode)(x, y) - result = max.(x, y) + result = max(x, y) return result end @@ -327,7 +327,6 @@ function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, cal op2 = pop!(stack) node = callbacks[elem]() push!(stack, (inputs) -> node(op1(inputs), op2(inputs))) - @show elem elseif get(arity_map, elem, 0) == 1 op = pop!(stack) node = callbacks[elem]() @@ -339,7 +338,6 @@ function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, cal end if nodes[elem] isa Number push!(stack, _ -> nodes[elem]) - #push!(stack, (inputs) -> ConstantNode(inputs_idx[elem])(inputs)) else push!(stack, (inputs) -> InputSelector(inputs_idx[elem])(inputs)) end diff --git a/test/runtests.jl b/test/runtests.jl index 3c1208b..ab370c9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ include("Entity_test.jl") include("Spb_core_test.jl") include("non_dom_sort_test.jl") -include("regression_wrapper_test.jl") \ No newline at end of file +include("regression_wrapper_test.jl") +include("tensor_ops_test.jl") \ No newline at end of file diff --git a/test/tensor_ops_test.jl b/test/tensor_ops_test.jl new file mode 100644 index 0000000..99323be --- /dev/null +++ b/test/tensor_ops_test.jl @@ -0,0 +1,250 @@ +include("../src/TensorOps.jl") + +using Test +using .TensorRegUtils +using Tensors +using OrderedCollections + + + +@testset "TensorRegUtils" begin + # Test setup + dim = 3 + t2 = rand(Tensor{2,dim}) + vec3 = rand(Tensor{1,dim}) + const_val = 2.0 + + data_ = Dict( + 5 => t2, + 6 => vec3, + 7 => const_val + ) + + arity_map = OrderedDict{Int8,Int}( + 1 => 2, # Addition + 2 => 2, # Multiplication + 3 => 2, # Double Contraction + 4 => 1 # Trace + ) + + callbacks = Dict{Int8,Any}( + Int8(1) => AdditionNode, + Int8(2) => MultiplicationNode, + Int8(3) => DoubleContractionNode, + Int8(4) => TraceNode + ) + + @testset "Basic Operations" begin + nodes = OrderedDict{Int8,Any}( + Int8(5) => InputSelector(1), + Int8(6) => InputSelector(2), + Int8(7) => const_val + ) + + # Scalar multiplication + rek_string = Int8[2, 6, 7] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + result = network(inputs) + @test result ≈ vec3 * const_val + + # Addition with dimension mismatch + rek_string = Int8[1, 5, 6] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test_throws DimensionMismatch network(inputs) + end + + @testset "Tensor Operations" begin + nodes = OrderedDict{Int8,Any}( + Int8(5) => InputSelector(1), + Int8(6) => InputSelector(2), + Int8(7) => const_val + ) + + # Double contraction + rek_string = Int8[3, 5, 5] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + result = network(inputs) + @test result ≈ dcontract(t2, t2) + + # Trace + rek_string = Int8[4, 5] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + result = network(inputs) + @test result ≈ tr(t2) + end + + @testset "Complex Expressions" begin + nodes = OrderedDict{Int8,Any}( + Int8(5) => InputSelector(1), + Int8(6) => InputSelector(2), + Int8(7) => const_val + ) + + # (vec3 * const_val) + vec3 + rek_string = Int8[1, 2, 6, 7, 6] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + result = network(inputs) + @test result ≈ (vec3 * const_val) + vec3 + + # (t2 * vec3) + const_val - should fail + rek_string = Int8[1, 2, 5, 6, 7] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test_throws MethodError network(inputs) + + # tr(t2) * const_val + tr(t2) + rek_string = Int8[1, 2, 4, 5, 7, 4, 5] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + result = network(inputs) + @test result ≈ tr(t2) * const_val + tr(t2) + end + + end + + @testset "All Node Operations" begin + dim = 3 + t2 = rand(Tensor{2,dim}) + t2_2 = rand(Tensor{2,dim}) + vec3 = rand(Vec{dim}) + const_val = 2.0 + + data_ = Dict(19 => t2, 20 => t2_2, 21 => vec3, 22 => const_val) + + arity_map = OrderedDict{Int8,Int}( + 1 => 2, 2 => 2, 3 => 2, 4 => 2, 5 => 2, 6 => 2, 7 => 2, # Binary ops + 8 => 1, 9 => 1, 10 => 1, 11 => 1, 12 => 1, 13 => 1, # Unary ops part 1 + 14 => 1, 15 => 1, 16 => 1, 17 => 2, 18 => 1 # Unary ops part 2 + DC + ) + + callbacks = Dict{Int8,Any}( + Int8(1) => AdditionNode, + Int8(2) => SubtractionNode, + Int8(3) => MultiplicationNode, + Int8(4) => DivisionNode, + Int8(5) => PowerNode, + Int8(6) => MinNode, + Int8(7) => MaxNode, + Int8(8) => InversionNode, + Int8(9) => TraceNode, + Int8(10) => DeterminantNode, + Int8(11) => SymmetricNode, + Int8(12) => SkewNode, + Int8(13) => VolumetricNode, + Int8(14) => DeviatricNode, + Int8(15) => TdotNode, + Int8(16) => DottNode, + Int8(17) => DoubleContractionNode, + Int8(18) => DeviatoricNode + ) + + @testset "Basic Node Operations" begin + nodes = OrderedDict{Int8,Any}( + Int8(19) => InputSelector(1), + Int8(20) => InputSelector(2), + Int8(21) => InputSelector(3), + Int8(22) => const_val + ) + + # Test 1: Addition + rek_string = Int8[1, 19, 20] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ t2 + t2_2 + + # Test 2: Subtraction + rek_string = Int8[2, 19, 20] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ t2 - t2_2 + + # Test 3: Multiplication with constant + rek_string = Int8[3, 19, 22] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ t2 * const_val + end + + @testset "Tensor Operations" begin + nodes = OrderedDict{Int8,Any}( + Int8(19) => InputSelector(1), + Int8(20) => InputSelector(2) + ) + + # Test 1: Double Contraction + rek_string = Int8[17, 19, 20] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ dcontract(t2, t2_2) + + # Test 2: Deviatoric + rek_string = Int8[14, 19] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ dev(t2) + + # Test 3: Trace + Symmetric + rek_string = Int8[9, 11, 19] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ tr(symmetric(t2)) + end + + @testset "Complex Compositions" begin + nodes = OrderedDict{Int8,Any}( + Int8(19) => InputSelector(1), + Int8(20) => InputSelector(2), + Int8(22) => const_val + ) + + # Test 1: (t2 ⊡ t2_2) * const_val + rek_string = Int8[3, 17, 19, 20, 22] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ dcontract(t2, t2_2) * const_val + + # Test 2: dev(symmetric(t2)) + t2_2 + rek_string = Int8[1, 14, 11, 19, 20] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ dev(symmetric(t2)) + t2_2 + + # Test 3: tr(t2) * tr(t2_2) + rek_string = Int8[3, 9, 19, 9, 20] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test network(inputs) ≈ tr(t2) * tr(t2_2) + end + + @testset "Error Cases" begin + nodes = OrderedDict{Int8,Any}( + Int8(19) => InputSelector(1), + Int8(21) => InputSelector(2), + Int8(22) => const_val + ) + + # Test 1: Dimension mismatch (tensor + vector) + rek_string = Int8[1, 19, 21] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test_throws DimensionMismatch network(inputs) + + # Test 2: Division by zero + data_zero = merge(data_, Dict(22 => 0.0)) + rek_string = Int8[4, 19, 22] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_zero[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test_throws MethodError network(inputs) + + # Test 3: Invalid double contraction + rek_string = Int8[17, 19, 21] + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + @test_throws MethodError network(inputs) + end + end \ No newline at end of file From 24de57561461d1f5cb70893d3b35a5ec899b692f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Fri, 24 Jan 2025 18:06:19 +1100 Subject: [PATCH 32/41] add new TensorRegressorWrapper + docu --- src/Entities.jl | 11 ++- src/GeneExpressionProgramming.jl | 9 ++- src/RegressionWrapper.jl | 132 +++++++++++++++++++++++++++---- src/TensorOps.jl | 72 ++++++++++++++--- src/Util.jl | 20 +---- test/tensor_ops_test.jl | 40 +++++----- 6 files changed, 211 insertions(+), 73 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index d999f78..4b03ff1 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -78,8 +78,11 @@ export generate_gene, compile_expression!, generate_chromosome, generate_populat export genetic_operations!, replicate, gene_inversion!, gene_mutation!, gene_one_point_cross_over!, gene_two_point_cross_over!, gene_fussion! include("Util.jl") +include("TensorOps.jl") + using .GepUtils +using .TensorRegUtils using OrderedCollections using DynamicExpressions @@ -227,11 +230,13 @@ struct Toolbox preamble_syms::Vector{Int8} len_preamble::Int8 operators_::Union{OperatorEnum,Nothing} + compile_function_::Function function Toolbox(gene_count::Int, head_len::Int, symbols::OrderedDict{Int8,Int8}, gene_connections::Vector{Int8}, callbacks::Dict, nodes::OrderedDict, gep_probs::Dict{String,AbstractFloat}; - unary_prob::Real=0.1, preamble_syms=Int8[], number_of_objectives::Int=1, operators_::Union{OperatorEnum,Nothing}=nothing) + unary_prob::Real=0.1, preamble_syms=Int8[], + number_of_objectives::Int=1, operators_::Union{OperatorEnum,Nothing}=nothing, function_complile::Function=compile_djl_datatype) fitness_reset= ( ntuple(_ -> Inf, number_of_objectives), @@ -245,7 +250,7 @@ struct Toolbox gen_start_indices = [gene_count + (gene_len * (i - 1)) for i in 1:gene_count] ensure_buffer_size!(head_len, gene_count) new(gene_count, head_len, symbols, gene_connections, headsyms, unary_syms, tailsyms, symbols, - callbacks, nodes, gen_start_indices, gep_probs, unary_prob, fitness_reset, preamble_syms, len_preamble, operators_) + callbacks, nodes, gen_start_indices, gep_probs, unary_prob, fitness_reset, preamble_syms, len_preamble, operators_, compile_djl_datatype) end end @@ -321,7 +326,7 @@ Fitness value or tuple if !chromosome.compiled || force_compile try expression = _karva_raw(chromosome) - expression_tree = compile_djl_datatype(expression, chromosome.toolbox.symbols, chromosome.toolbox.callbacks, + expression_tree = chromosome.toolbox.compile_function_(expression, chromosome.toolbox.symbols, chromosome.toolbox.callbacks, chromosome.toolbox.nodes, max(chromosome.toolbox.len_preamble,1)) chromosome.compiled_function = expression_tree chromosome.expression_raw = expression diff --git a/src/GeneExpressionProgramming.jl b/src/GeneExpressionProgramming.jl index ecf9152..8b01a4c 100644 --- a/src/GeneExpressionProgramming.jl +++ b/src/GeneExpressionProgramming.jl @@ -114,7 +114,9 @@ import .GepUtils: HistoryRecorder, OptimizationHistory, get_history_arrays, - train_test_split + train_test_split, + ARITY_LIB_COMMON, + FUNCTION_LIB_COMMON # Import selection mechanisms import .EvoSelection: @@ -179,6 +181,7 @@ import .GepEntities: # Import regression wrapper functionality import .RegressionWrapper: GepRegressor, + GepTensorRegressor, fit!, list_all_functions, list_all_arity, @@ -209,12 +212,12 @@ export get_feature_dims_json, get_target_dim_json, retrieve_coeffs_based_on_simi export Chromosome, Toolbox, fitness, set_fitness! export generate_gene, compile_expression!, generate_chromosome, generate_population export genetic_operations! -export GepRegressor, fit! +export GepRegressor,GepTensorRegressor, fit! export list_all_functions, list_all_arity, list_all_forward_handlers export list_all_backward_handlers, list_all_genetic_params export set_function!, set_arity!, set_forward_handler!, set_backward_handler! export update_function! - +export ARITY_LIB_COMMON, FUNCTION_LIB_COMMON end diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index be11250..6b9344b 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -121,7 +121,7 @@ See also: module RegressionWrapper -export GepRegressor +export GepRegressor, GepTensorRegressor export create_function_entries, create_feature_entries, create_constants_entries, create_physical_operations export GENE_COMMON_PROBS, FUNCTION_LIB_BACKWARD_COMMON, FUNCTION_LIB_FORWARD_COMMON export fit! @@ -138,6 +138,7 @@ include("PhyConstants.jl") include("Sbp.jl") include("Selection.jl") include("Util.jl") +include("TensorOps.jl") using .GepEntities @@ -147,6 +148,7 @@ using .EvoSelection using .GepRegression using .SBPUtils using .GepUtils +using .TensorRegUtils using DynamicExpressions using OrderedCollections using LinearAlgebra @@ -454,6 +456,7 @@ Create a Gene Expression Programming regressor for symbolic regression. - `preamble_syms::Vector{Symbol}=Symbol[]`: Preamble symbols - `max_permutations_lib::Int=10000`: Maximum permutations for dimension library - `rounds::Int=4`: Rounds for dimension library creation +- `number_of_objectives::Int=1`: Defines the number of objectives considered by the search """ mutable struct GepRegressor toolbox_::Toolbox @@ -541,6 +544,105 @@ mutable struct GepRegressor end end +""" + GepTensorRegressor + +A Gene Expression Programming (GEP) regressor that evolves higher order mathematical expressions (e.g tensor-based). + +# Fields +- `toolbox_::Toolbox`: Contains configuration and operators for GEP evolution +- `best_models_::Union{Nothing,Vector{Chromosome}}`: Best models found during evolution +- `fitness_history_::Any`: History of fitness values during training + +# Constructor + GepTensorRegressor(feature_amount::Int; kwargs...) + +Create a new GEP tensor regressor with specified number of input features. + +# Arguments +- `feature_amount::Int`: Number of input features to use in the regression + +# Keyword Arguments +- `entered_non_terminals::Vector{Symbol}=[:+, :-, :*, :/]`: Available mathematical operators +- `entered_terminal_nums::Vector{<:AbstractFloat}=[0.0, 0.5]`: Constants available as terminals +- `gene_connections::Vector{Symbol}=[:+, :-, :*, :/]`: Operators for connecting genes +- `rnd_count::Int=1`: Number of random constant terminals to generate +- `gene_count::Int=3`: Number of genes in each chromosome +- `head_len::Int=6`: Length of the head section in each gene +- `number_of_objectives::Int=1`: Number of optimization objectives + +The regressor uses GEP to evolve tensor-based mathematical expressions that map input features +to target values. It supports multiple genes connected by operators and can optimize for +multiple objectives. + +# Implementation Details +- Uses InputSelector nodes for features +- Combines fixed and random constant terminals +- Maps operators to TENSOR_NODES callbacks +- Uses TENSOR_NODES_ARITY for operator arity +- Compiles expressions to Flux networks via compile_to_flux_network +""" +mutable struct GepTensorRegressor + toolbox_::Toolbox + best_models_::Union{Nothing,Vector{Chromosome}} + fitness_history_::Any + + + function GepTensorRegressor(feature_amount::Int; + entered_non_terminals::Vector{Symbol}=[:+, :-, :*, :/], + entered_terminal_nums::Vector{<:AbstractFloat}=[0.0, 0.5], + gene_connections::Vector{Symbol}=[:+, :-, :*, :/], + rnd_count::Int=1, + gene_count::Int=3, + head_len::Int=6, + number_of_objectives::Int=1 + ) + #Creating the feature Nodes -> asuming a data dict pointing to + cur_idx = Int8(1) + nodes = OrderedDict{Int8,Any}() + utilized_symbols = SymbolDict() + callbacks = Dict{Int8,Any}() + gene_connections_ = Int8[] + for _ in 1:feature_amount + nodes[cur_idx] = InputSelector(cur_idx) + utilized_symbols[cur_idx] = Int8(0) + cur_idx += 1 + end + + #Creating the const_nodes + for elem in entered_terminal_nums + nodes[cur_idx] = elem + utilized_symbols[cur_idx] = Int8(0) + cur_idx += 1 + end + + for _ in 1:rnd_count + nodes[cur_idx] = rand() + utilized_symbols[cur_idx] = Int8(0) + cur_idx += 1 + end + + #callback - index => function + for elem in entered_non_terminals + callbacks[cur_idx] = TENSOR_NODES[elem] + utilized_symbols[cur_idx] = TENSOR_NODES_ARITY[elem] + if elem in gene_connections + push!(gene_connections_) + end + cur_idx += 1 + end + + + toolbox = GepRegression.GepEntities.Toolbox(gene_count, head_len, utilized_symbols, gene_connections_, + callbacks, nodes, GENE_COMMON_PROBS; number_of_objectives=number_of_objectives, + operators_=nothing, function_complile=compile_to_flux_network) + + obj = new() + obj.toolbox_ = toolbox + return obj + end +end + """ fit!(regressor::GepRegressor, epochs::Int, population_size::Int, x_train::AbstractArray, @@ -709,7 +811,7 @@ Make predictions using the trained regressor. - `ensemble::Bool=false`: Whether to use ensemble predictions # Returns -- Predicted values for the input features +- Predicted values for the input features -> only employable when using compile_djl_datatype """ function (regressor::GepRegressor)(x_data::AbstractArray; ensemble::Bool=false) return regressor.best_models_[1].compiled_function(x_data, regressor.operators_) @@ -738,11 +840,11 @@ println(sin_info.arity) # 1 """ function list_all_functions() return Dict(sym => ( - function_=_FUNCTION_LIB_COMMON[sym], - arity=_ARITY_LIB_COMMON[sym], - forward_handler=_FUNCTION_LIB_FORWARD_COMMON[sym], - backward_handler=_FUNCTION_LIB_BACKWARD_COMMON[sym] - ) for sym in keys(_FUNCTION_LIB_COMMON)) + function_=FUNCTION_LIB_COMMON[sym], + arity=ARITY_LIB_COMMON[sym], + forward_handler=FUNCTION_LIB_FORWARD_COMMON[sym], + backward_handler=FUNCTION_LIB_BACKWARD_COMMON[sym] + ) for sym in keys(FUNCTION_LIB_COMMON)) end """ @@ -837,8 +939,8 @@ set_function!(:sin, new_sin_implementation) - ArgumentError if the function symbol doesn't exist in the library """ function set_function!(sym::Symbol, func::Function) - haskey(_FUNCTION_LIB_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) - _FUNCTION_LIB_COMMON[sym] = func + haskey(_UNCTION_LIB_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) + FUNCTION_LIB_COMMON[sym] = func return nothing end @@ -860,9 +962,9 @@ set_arity!(:custom_func, 2) - ArgumentError if the function symbol doesn't exist or arity is invalid """ function set_arity!(sym::Symbol, arity::Int8) - haskey(_ARITY_LIB_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) + haskey(ARITY_LIB_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) arity in (1, 2) || throw(ArgumentError("Arity must be 1 or 2")) - _ARITY_LIB_COMMON[sym] = arity + ARITY_LIB_COMMON[sym] = arity return nothing end @@ -884,8 +986,8 @@ set_forward_handler!(:custom_func, zero_unit_forward) - ArgumentError if the function symbol doesn't exist """ function set_forward_handler!(sym::Symbol, handler::Function) - haskey(_FUNCTION_LIB_FORWARD_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) - _FUNCTION_LIB_FORWARD_COMMON[sym] = handler + haskey(FUNCTION_LIB_FORWARD_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) + FUNCTION_LIB_FORWARD_COMMON[sym] = handler return nothing end @@ -907,8 +1009,8 @@ set_backward_handler!(:custom_func, zero_unit_backward) - ArgumentError if the function symbol doesn't exist """ function set_backward_handler!(sym::Symbol, handler::Function) - haskey(_FUNCTION_LIB_BACKWARD_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) - _FUNCTION_LIB_BACKWARD_COMMON[sym] = handler + haskey(FUNCTION_LIB_BACKWARD_COMMON, sym) || throw(ArgumentError("Function $sym not found in library")) + FUNCTION_LIB_BACKWARD_COMMON[sym] = handler return nothing end diff --git a/src/TensorOps.jl b/src/TensorOps.jl index 743085f..5c86dcb 100644 --- a/src/TensorOps.jl +++ b/src/TensorOps.jl @@ -14,9 +14,11 @@ export VolumetricNode, DeviatricNode, TdotNode, DottNode export DoubleContractionNode, DeviatoricNode export ConstantNode, UnaryNode -# Core functions export compile_to_flux_network +export TENSOR_NODES, TENSOR_NODES_ARITY + + abstract type OperationNode end struct InputSelector @@ -154,7 +156,7 @@ end const NODES = [ AdditionNode, - SubtractionNode, + SubtractionNode, MultiplicationNode, DivisionNode, PowerNode, @@ -176,7 +178,7 @@ const NODES = [ ] for T in NODES - @eval Flux.Functors.@functor $T + @eval Flux.Functors.@functor $T end @inline function (l::AdditionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) @@ -211,22 +213,22 @@ end end @inline function (l::MultiplicationNode)(x::Number, y::Number) - result = y * x + result = y * x return result end @inline function (l::MultiplicationNode)(x::Union{Tensor,SymmetricTensor}, y::Number) - result = y * x + result = y * x return result end @inline function (l::MultiplicationNode)(x::Number, y::Union{Tensor,SymmetricTensor}) - result = y * x + result = y * x return result end @inline function (l::MultiplicationNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) - result = dot(x,y) + result = dot(x, y) return result end @@ -237,7 +239,7 @@ end @inline function (l::PowerNode)(x::Union{Tensor,SymmetricTensor,Number}, y::Number) - result = x ^ y + result = x^y return result end @@ -317,9 +319,9 @@ end return result end -function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict; use_cuda::Bool=false) +function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict, pre_len::Int) stack = [] - inputs_idx = Dict{Int8, Int8}() + inputs_idx = Dict{Int8,Int8}() idx = 1 for elem in reverse(rek_string) if get(arity_map, elem, 0) == 2 @@ -333,8 +335,8 @@ function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, cal push!(stack, (inputs) -> node(op(inputs))) else if get(inputs_idx, elem, 0) == 0 - inputs_idx[elem] = idx - idx +=1 + inputs_idx[elem] = idx + idx += 1 end if nodes[elem] isa Number push!(stack, _ -> nodes[elem]) @@ -342,11 +344,55 @@ function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, cal push!(stack, (inputs) -> InputSelector(inputs_idx[elem])(inputs)) end end - + end return Chain(pop!(stack)), inputs_idx end + +const TENSOR_NODES = Dict{Symbol,Type}( + :+ => AdditionNode, + :- => SubtractionNode, + :* => MultiplicationNode, + :/ => DivisionNode, + :^ => PowerNode, + :min => MinNode, + :max => MaxNode, + :inv => InversionNode, + :tr => TraceNode, + :det => DeterminantNode, + :symmetric => SymmetricNode, + :skew => SkewNode, + :vol => VolumetricNode, + :dev => DeviatricNode, + :tdot => TdotNode, + :dott => DottNode, + :dcontract => DoubleContractionNode, + :deviator => DeviatoricNode +) + +const TENSOR_NODES_ARITY = Dict{Symbol,Int8}( + :+ => 2, + :- => 2, + :* => 2, + :/ => 2, + :^ => 2, + :min => 2, + :max => 2, + :inv => 1, + :tr => 1, + :det => 1, + :symmetric => 1, + :skew => 1, + :vol => 1, + :dev => 1, + :tdot => 1, + :dott => 1, + :dcontract => 2, + :deviator => 1 +) + + """ Later on cuda Module diff --git a/src/Util.jl b/src/Util.jl index 0dcfe97..59a2dc9 100644 --- a/src/Util.jl +++ b/src/Util.jl @@ -181,15 +181,7 @@ const FUNCTION_LIB_COMMON = Dict{Symbol,Function}( :asinh => asinh, :acosh => acosh, :atanh => atanh, :sqr => sqr, - :sqrt => sqrt, :sign => sign, - - - # Tensor specific operations -> never use in scalar regression tasks - documenation: https://ferrite-fem.github.io/Tensors.jl/stable/man/other_operators/ - :inv => inv, :tr => tr, :det => det, - :symmetric => symmetric, :skew => skew, - :vol => vol, :dev => dev, - :dot => dot, :dcontract => dcontract, - :tdot => tdot, :dott => dott + :sqrt => sqrt, :sign => sign ) """ @@ -230,14 +222,7 @@ const ARITY_LIB_COMMON = Dict{Symbol,Int8}( :acosh => 1, :atanh => 1, :sqrt => 1, - :sqr => 1, - - # Tensor specific operations -> never use in scalar regression tasks - documenation: https://ferrite-fem.github.io/Tensors.jl/stable/man/other_operators/ - :inv => 1, :tr => 1, :det => 1, - :symmetric => 1, :skew => 1, - :vol => 1, :dev => 1, - :dot => 2, :dcontract => 2, - :tdot => 1, :dott => 1 + :sqr => 1 ) struct OptimizationHistory{T<:Union{AbstractFloat,Tuple}} @@ -550,7 +535,6 @@ result = compile_djl_datatype(rek_string, arity_map, callbacks, nodes) See also: [`DynamicExpressions.Node`](@ref) """ function compile_djl_datatype(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict, pre_len::Int) - #just let it fail when it becomes invalid, because then the equation is not that use full stack = [] for elem in reverse(rek_string[pre_len:end]) if get(arity_map, elem, 0) == 2 diff --git a/test/tensor_ops_test.jl b/test/tensor_ops_test.jl index 99323be..b5c87c9 100644 --- a/test/tensor_ops_test.jl +++ b/test/tensor_ops_test.jl @@ -5,8 +5,6 @@ using .TensorRegUtils using Tensors using OrderedCollections - - @testset "TensorRegUtils" begin # Test setup dim = 3 @@ -43,14 +41,14 @@ using OrderedCollections # Scalar multiplication rek_string = Int8[2, 6, 7] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) result = network(inputs) @test result ≈ vec3 * const_val # Addition with dimension mismatch rek_string = Int8[1, 5, 6] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test_throws DimensionMismatch network(inputs) end @@ -64,14 +62,14 @@ using OrderedCollections # Double contraction rek_string = Int8[3, 5, 5] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) result = network(inputs) @test result ≈ dcontract(t2, t2) # Trace rek_string = Int8[4, 5] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) result = network(inputs) @test result ≈ tr(t2) @@ -86,20 +84,20 @@ using OrderedCollections # (vec3 * const_val) + vec3 rek_string = Int8[1, 2, 6, 7, 6] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) result = network(inputs) @test result ≈ (vec3 * const_val) + vec3 # (t2 * vec3) + const_val - should fail rek_string = Int8[1, 2, 5, 6, 7] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test_throws MethodError network(inputs) # tr(t2) * const_val + tr(t2) rek_string = Int8[1, 2, 4, 5, 7, 4, 5] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) result = network(inputs) @test result ≈ tr(t2) * const_val + tr(t2) @@ -153,19 +151,19 @@ using OrderedCollections # Test 1: Addition rek_string = Int8[1, 19, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ t2 + t2_2 # Test 2: Subtraction rek_string = Int8[2, 19, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ t2 - t2_2 # Test 3: Multiplication with constant rek_string = Int8[3, 19, 22] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ t2 * const_val end @@ -178,19 +176,19 @@ using OrderedCollections # Test 1: Double Contraction rek_string = Int8[17, 19, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ dcontract(t2, t2_2) # Test 2: Deviatoric rek_string = Int8[14, 19] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ dev(t2) # Test 3: Trace + Symmetric rek_string = Int8[9, 11, 19] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ tr(symmetric(t2)) end @@ -204,19 +202,19 @@ using OrderedCollections # Test 1: (t2 ⊡ t2_2) * const_val rek_string = Int8[3, 17, 19, 20, 22] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ dcontract(t2, t2_2) * const_val # Test 2: dev(symmetric(t2)) + t2_2 rek_string = Int8[1, 14, 11, 19, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ dev(symmetric(t2)) + t2_2 # Test 3: tr(t2) * tr(t2_2) rek_string = Int8[3, 9, 19, 9, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test network(inputs) ≈ tr(t2) * tr(t2_2) end @@ -230,20 +228,20 @@ using OrderedCollections # Test 1: Dimension mismatch (tensor + vector) rek_string = Int8[1, 19, 21] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test_throws DimensionMismatch network(inputs) # Test 2: Division by zero data_zero = merge(data_, Dict(22 => 0.0)) rek_string = Int8[4, 19, 22] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_zero[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test_throws MethodError network(inputs) # Test 3: Invalid double contraction rek_string = Int8[17, 19, 21] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes) + network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) @test_throws MethodError network(inputs) end From 6f87d123df4e70b87ca88a4dd57dfe9bf048e9c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Fri, 24 Jan 2025 18:20:56 +1100 Subject: [PATCH 33/41] add some docu --- src/RegressionWrapper.jl | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index 6b9344b..403b47a 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -797,7 +797,30 @@ function fit!(regressor::GepRegressor, epochs::Int, population_size::Int, loss_f regressor.fitness_history_ = history end +function fit!(regressor::GepTensorRegressor, epochs::Int, population_size::Int, loss_function::Function; + hof::Int=3, + break_condition::Union{Function,Nothing}=nothing +) + + evalStrat = GenericRegressionStrategy( + regressor.operators_, + length(regressor.toolbox_.fitness_reset[1]), + loss_function; + secOptimizer=nothing, + break_condition=break_condition + ) + best, history = runGep(epochs, + population_size, + regressor.toolbox_, + evalStrat; + hof=hof, + tourni_size=max(Int(ceil(population_size * 0.03)), 3) + ) + + regressor.best_models_ = best + regressor.fitness_history_ = history +end """ (regressor::GepRegressor)(x_data::AbstractArray; ensemble::Bool=false) From ef5fa9aafc739becb4b3c1d44747fc5f74a29f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Fri, 24 Jan 2025 22:01:51 +1100 Subject: [PATCH 34/41] simplify inputs --- src/TensorOps.jl | 14 +++------ test/tensor_ops_test.jl | 68 ++++++++++++++++------------------------- 2 files changed, 32 insertions(+), 50 deletions(-) diff --git a/src/TensorOps.jl b/src/TensorOps.jl index 5c86dcb..770ed8e 100644 --- a/src/TensorOps.jl +++ b/src/TensorOps.jl @@ -321,8 +321,6 @@ end function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict, pre_len::Int) stack = [] - inputs_idx = Dict{Int8,Int8}() - idx = 1 for elem in reverse(rek_string) if get(arity_map, elem, 0) == 2 op1 = pop!(stack) @@ -334,19 +332,17 @@ function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, cal node = callbacks[elem]() push!(stack, (inputs) -> node(op(inputs))) else - if get(inputs_idx, elem, 0) == 0 - inputs_idx[elem] = idx - idx += 1 - end if nodes[elem] isa Number - push!(stack, _ -> nodes[elem]) + num = nodes[elem] + push!(stack, _ -> num) else - push!(stack, (inputs) -> InputSelector(inputs_idx[elem])(inputs)) + idx = nodes[elem].idx + push!(stack, (inputs) -> InputSelector(idx)(inputs)) end end end - return Chain(pop!(stack)), inputs_idx + return Chain(pop!(stack)) end diff --git a/test/tensor_ops_test.jl b/test/tensor_ops_test.jl index b5c87c9..968fa6f 100644 --- a/test/tensor_ops_test.jl +++ b/test/tensor_ops_test.jl @@ -17,7 +17,9 @@ using OrderedCollections 6 => vec3, 7 => const_val ) - + inputs = (t2,vec3,const_val) + + arity_map = OrderedDict{Int8,Int}( 1 => 2, # Addition 2 => 2, # Multiplication @@ -41,15 +43,13 @@ using OrderedCollections # Scalar multiplication rek_string = Int8[2, 6, 7] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) result = network(inputs) @test result ≈ vec3 * const_val # Addition with dimension mismatch rek_string = Int8[1, 5, 6] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) @test_throws DimensionMismatch network(inputs) end @@ -62,15 +62,13 @@ using OrderedCollections # Double contraction rek_string = Int8[3, 5, 5] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) result = network(inputs) @test result ≈ dcontract(t2, t2) # Trace rek_string = Int8[4, 5] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) result = network(inputs) @test result ≈ tr(t2) end @@ -84,21 +82,19 @@ using OrderedCollections # (vec3 * const_val) + vec3 rek_string = Int8[1, 2, 6, 7, 6] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) + result = network(inputs) @test result ≈ (vec3 * const_val) + vec3 # (t2 * vec3) + const_val - should fail rek_string = Int8[1, 2, 5, 6, 7] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) @test_throws MethodError network(inputs) # tr(t2) * const_val + tr(t2) rek_string = Int8[1, 2, 4, 5, 7, 4, 5] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) result = network(inputs) @test result ≈ tr(t2) * const_val + tr(t2) end @@ -113,6 +109,8 @@ using OrderedCollections const_val = 2.0 data_ = Dict(19 => t2, 20 => t2_2, 21 => vec3, 22 => const_val) + + inputs = (t2, t2_2, vec3, const_val) arity_map = OrderedDict{Int8,Int}( 1 => 2, 2 => 2, 3 => 2, 4 => 2, 5 => 2, 6 => 2, 7 => 2, # Binary ops @@ -151,20 +149,17 @@ using OrderedCollections # Test 1: Addition rek_string = Int8[1, 19, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ t2 + t2_2 # Test 2: Subtraction rek_string = Int8[2, 19, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ t2 - t2_2 # Test 3: Multiplication with constant rek_string = Int8[3, 19, 22] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ t2 * const_val end @@ -176,20 +171,17 @@ using OrderedCollections # Test 1: Double Contraction rek_string = Int8[17, 19, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ dcontract(t2, t2_2) # Test 2: Deviatoric rek_string = Int8[14, 19] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ dev(t2) # Test 3: Trace + Symmetric rek_string = Int8[9, 11, 19] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ tr(symmetric(t2)) end @@ -202,47 +194,41 @@ using OrderedCollections # Test 1: (t2 ⊡ t2_2) * const_val rek_string = Int8[3, 17, 19, 20, 22] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ dcontract(t2, t2_2) * const_val # Test 2: dev(symmetric(t2)) + t2_2 rek_string = Int8[1, 14, 11, 19, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ dev(symmetric(t2)) + t2_2 # Test 3: tr(t2) * tr(t2_2) rek_string = Int8[3, 9, 19, 9, 20] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ tr(t2) * tr(t2_2) end @testset "Error Cases" begin nodes = OrderedDict{Int8,Any}( Int8(19) => InputSelector(1), - Int8(21) => InputSelector(2), + Int8(21) => InputSelector(3), Int8(22) => const_val ) # Test 1: Dimension mismatch (tensor + vector) rek_string = Int8[1, 19, 21] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test_throws DimensionMismatch network(inputs) # Test 2: Division by zero data_zero = merge(data_, Dict(22 => 0.0)) rek_string = Int8[4, 19, 22] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_zero[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test_throws MethodError network(inputs) # Test 3: Invalid double contraction rek_string = Int8[17, 19, 21] - network, inputs_ids = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - inputs = Tuple(data_[k] for (k,v) in sort(collect(inputs_ids), by=x->x[2])) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test_throws MethodError network(inputs) end - end \ No newline at end of file + end From c0da113e5110f74ab109c2ac319b0f46cc09f552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sat, 25 Jan 2025 12:21:34 +1100 Subject: [PATCH 35/41] add benchmark --- benchmark/Benchmark.md | 90 ++++++++++++++++++++++++++++++++++ benchmark/benchmark_djl_nn.jl | 81 +++++++++++++++++++++++++++++++ src/TensorOps.jl | 78 ++++++++++-------------------- test/tensor_ops_test.jl | 91 ++++++++++++++++++----------------- 4 files changed, 243 insertions(+), 97 deletions(-) create mode 100644 benchmark/Benchmark.md create mode 100644 benchmark/benchmark_djl_nn.jl diff --git a/benchmark/Benchmark.md b/benchmark/Benchmark.md new file mode 100644 index 0000000..d7575ee --- /dev/null +++ b/benchmark/Benchmark.md @@ -0,0 +1,90 @@ +# Tensor Operation Benchmarks + +Comparing tensor computations using DynamicExpressions.jl vs custom Flux network. + +## Usage + +```bash +export JULIA_NUM_THREADS=1 +``` + +# Evaluating Performance of DynamicExpressions +- example from: https://github.com/SymbolicML/DynamicExpressions.jl (adapted for utilizing Tensors as Dtype) + +```julia +using DynamicExpressions +using DynamicExpressions: @declare_expression_operator +using BenchmarkTools +using LinearAlgebra + +# Operations +vec_add(x::Tensor, y::Tensor) = @fastmath x + y +vec_square(x::Tensor) = @fastmath dot(x,x) + +@declare_expression_operator(vec_add, 2) +@declare_expression_operator(vec_square, 1) + +# Build expression +operators = GenericOperatorEnum( + binary_operators=[vec_add], + unary_operators=[vec_square] +) +variable_names = ["x1"] +c1 = Expression(Node{T}(; val=ones(Tensor{2,3})); operators, variable_names); +expression = vec_add(vec_add(vec_square(c1), c1), c1); +X = ones(Tensor{2,3}); + +#Evalutate the expression: + +tests_n = 100000 +@show "Benchmark expression" +expression(X) # [[5.0 5.0 5.0], [5.0 5.0 5.0], [5.0 5.0 5.0]] +@btime for _ in 1:tests_n + expression(X) +end + +# 83.021 ms (1798979 allocations: 187.67 MiB) +``` + +# Evaluating performance when generated with Flux + +```julia +# create the inputs for Flux +c1_ = ones(Tensor{2,3}); +inputs = (c1_,); + +# create the arity map +arity_map = OrderedDict{Int8,Int}( + 1 => 2, # Addition + 2 => 2 # Multiplication +); + +#assign the callbacks +callbacks = Dict{Int8,Any}( + Int8(1) => AdditionNode, + Int8(2) => MultiplicationNode +); + +#define nodes +nodes = OrderedDict{Int8,Any}( + Int8(5) => InputSelector(1) +); + +rek_string = Int8[1, 1, 2, 5, 5, 5, 5]; +network = TensorRegUtils.compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0); +@show "Benchmark network" +result = network(inputs) # [[5.0 5.0 5.0], [5.0 5.0 5.0], [5.0 5.0 5.0]] +@btime for _ in 1:tests_n + result = network(inputs) +end + +#11.703 ms (998979 allocations: 59.49 MiB) + +``` + +## Conclusion +- Flux in handling higher dimension around 7.5 times faster +- Flux uses around 3.8 times less memory resources + + + diff --git a/benchmark/benchmark_djl_nn.jl b/benchmark/benchmark_djl_nn.jl new file mode 100644 index 0000000..23d9e1b --- /dev/null +++ b/benchmark/benchmark_djl_nn.jl @@ -0,0 +1,81 @@ +using DynamicExpressions +using DynamicExpressions: @declare_expression_operator +using BenchmarkTools +using LinearAlgebra + +include("../src/TensorOps.jl") +using .TensorRegUtils +using Tensors +using OrderedCollections +using Flux + + +""" +Benchmark for comparing higher dim structures for tensor regression - run test with - export JULIA_NUM_THREADS=1 +example from: https://github.com/SymbolicML/DynamicExpressions.jl with changes according to utilize tensors +""" + + +T = Union{Float64,Vector{Float64},Tensor} +vec_add(x::Tensor, y::Tensor) = @fastmath x + y; +vec_square(x::Tensor) = @fastmath dot(x,x); + + +@declare_expression_operator(vec_add, 2); +@declare_expression_operator(vec_square, 1); + + +operators = GenericOperatorEnum(; binary_operators=[vec_add], unary_operators=[vec_square]); + +# Construct the expression: +variable_names = ["x1"] +c1 = Expression(Node{T}(; val=ones(Tensor{2,3})); operators, variable_names); +expression = vec_add(vec_add(vec_square(c1), c1), c1); + +X = ones(Tensor{2,3}); + + +# create the inputs for Flux +c1_ = ones(Tensor{2,3}); +inputs = (c1_,); + +# create the arity map +arity_map = OrderedDict{Int8,Int}( + 1 => 2, # Addition + 2 => 2 # Multiplication +); + +#assign the callbacks +callbacks = Dict{Int8,Any}( + Int8(1) => AdditionNode, + Int8(2) => MultiplicationNode +); + +#define nodes +nodes = OrderedDict{Int8,Any}( + Int8(5) => InputSelector(1) +); + +# Evaluate - expression +# Solution => [[5.0 5.0 5.0], [5.0 5.0 5.0], [5.0 5.0 5.0]] +tests_n = 100000 +@show "Benchmark expression" +expression(X) +@btime for _ in 1:tests_n + expression(X) +end + +#83.021 ms (1798979 allocations: 187.67 MiB) + +rek_string = Int8[1, 1, 2, 5, 5, 5, 5]; +network = TensorRegUtils.compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0); +@show "Benchmark network" +result = network(inputs) +@btime for _ in 1:tests_n + result = network(inputs) +end + +#11.703 ms (998979 allocations: 59.49 MiB) + + +#Conclusion -> 4 times faster than DynamicExpressions.jl for such structures \ No newline at end of file diff --git a/src/TensorOps.jl b/src/TensorOps.jl index 770ed8e..4f7bcc4 100644 --- a/src/TensorOps.jl +++ b/src/TensorOps.jl @@ -182,131 +182,106 @@ for T in NODES end @inline function (l::AdditionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) - result = x + y - return result + @fastmath return x + y end @inline function (l::AdditionNode)(x::Vector, y::Vector) - result = x + y - return result + @fastmath return x + y end @inline function (l::AdditionNode)(x::Number, y::Number) - result = x + y - return result + @fastmath return x + y end @inline function (l::SubtractionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) - result = x - y - return result + @fastmath return x - y end @inline function (l::SubtractionNode)(x::Vector, y::Vector) - result = x - y - return result + @fastmath return x - y end @inline function (l::SubtractionNode)(x::Number, y::Number) - result = x - y - return result + @fastmath return x - y end @inline function (l::MultiplicationNode)(x::Number, y::Number) - result = y * x - return result + @fastmath return y * x end @inline function (l::MultiplicationNode)(x::Union{Tensor,SymmetricTensor}, y::Number) - result = y * x - return result + @fastmath return y * x end @inline function (l::MultiplicationNode)(x::Number, y::Union{Tensor,SymmetricTensor}) - result = y * x - return result + @fastmath return y * x end @inline function (l::MultiplicationNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) - result = dot(x, y) - return result + @fastmath return dot(x, y) end @inline function (l::DivisionNode)(x::Union{Tensor,SymmetricTensor}, y::Vector) - result = x / y - return result + @fastmath return x / y end @inline function (l::PowerNode)(x::Union{Tensor,SymmetricTensor,Number}, y::Number) - result = x^y - return result + @fastmath return x^y end @inline function (l::DoubleContractionNode)(x, y) - result = dcontract(x, y) - return result + @fastmath return dcontract(x, y) end @inline function (l::DeviatoricNode)(x) - result = dev(x) - return result + @fastmath return dev(x) end @inline function (l::MinNode)(x, y) - result = min(x, y) - return result + @fastmath return min(x, y) end @inline function (l::MaxNode)(x, y) - result = max(x, y) - return result + @fastmath return max(x, y) end @inline function (l::InversionNode)(x) - result = inv(x) - return result + @fastmath return inv(x) end @inline function (l::TraceNode)(x) - result = tr(x) - return result + @fastmath return tr(x) end @inline function (l::DeterminantNode)(x) - result = det(x) - return result + @fastmath return det(x) end @inline function (l::SymmetricNode)(x) - result = symmetric(x) - return result + @fastmath return symmetric(x) end @inline function (l::SkewNode)(x) - result = skew(x) - return result + @fastmath return skew(x) end @inline function (l::VolumetricNode)(x) - result = vol(x) - return result + @fastmath return vol(x) end @inline function (l::DeviatricNode)(x) - result = dev(x) - return result + @fastmath return dev(x) end @inline function (l::TdotNode)(x) - result = tdot(x) - return result + @fastmath return tdot(x) end @inline function (l::DottNode)(x) - result = dott(x) - return result + @fastmath return dott(x) end @inline function (l::ConstantNode)(x) @@ -315,8 +290,7 @@ end @inline function (l::UnaryNode)(x) - result = l.operation.(x) - return result + @fastmath return l.operation.(x) end function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict, pre_len::Int) diff --git a/test/tensor_ops_test.jl b/test/tensor_ops_test.jl index 968fa6f..fc8de25 100644 --- a/test/tensor_ops_test.jl +++ b/test/tensor_ops_test.jl @@ -1,9 +1,10 @@ include("../src/TensorOps.jl") - +using BenchmarkTools using Test using .TensorRegUtils using Tensors using OrderedCollections +using Flux @testset "TensorRegUtils" begin # Test setup @@ -11,13 +12,13 @@ using OrderedCollections t2 = rand(Tensor{2,dim}) vec3 = rand(Tensor{1,dim}) const_val = 2.0 - + data_ = Dict( 5 => t2, 6 => vec3, 7 => const_val ) - inputs = (t2,vec3,const_val) + inputs = (t2, vec3, const_val) arity_map = OrderedDict{Int8,Int}( @@ -26,98 +27,98 @@ using OrderedCollections 3 => 2, # Double Contraction 4 => 1 # Trace ) - + callbacks = Dict{Int8,Any}( Int8(1) => AdditionNode, Int8(2) => MultiplicationNode, Int8(3) => DoubleContractionNode, Int8(4) => TraceNode ) - + @testset "Basic Operations" begin nodes = OrderedDict{Int8,Any}( Int8(5) => InputSelector(1), Int8(6) => InputSelector(2), Int8(7) => const_val ) - + # Scalar multiplication rek_string = Int8[2, 6, 7] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) result = network(inputs) @test result ≈ vec3 * const_val - + # Addition with dimension mismatch rek_string = Int8[1, 5, 6] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test_throws DimensionMismatch network(inputs) end - + @testset "Tensor Operations" begin nodes = OrderedDict{Int8,Any}( Int8(5) => InputSelector(1), Int8(6) => InputSelector(2), Int8(7) => const_val ) - + # Double contraction rek_string = Int8[3, 5, 5] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) result = network(inputs) @test result ≈ dcontract(t2, t2) - + # Trace rek_string = Int8[4, 5] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) result = network(inputs) @test result ≈ tr(t2) end - + @testset "Complex Expressions" begin nodes = OrderedDict{Int8,Any}( Int8(5) => InputSelector(1), Int8(6) => InputSelector(2), Int8(7) => const_val ) - + # (vec3 * const_val) + vec3 rek_string = Int8[1, 2, 6, 7, 6] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) - + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) + result = network(inputs) @test result ≈ (vec3 * const_val) + vec3 - + # (t2 * vec3) + const_val - should fail rek_string = Int8[1, 2, 5, 6, 7] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test_throws MethodError network(inputs) - + # tr(t2) * const_val + tr(t2) rek_string = Int8[1, 2, 4, 5, 7, 4, 5] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes,0) + network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) result = network(inputs) @test result ≈ tr(t2) * const_val + tr(t2) end - - end - @testset "All Node Operations" begin +end + +@testset "All Node Operations" begin dim = 3 t2 = rand(Tensor{2,dim}) t2_2 = rand(Tensor{2,dim}) vec3 = rand(Vec{dim}) const_val = 2.0 - + data_ = Dict(19 => t2, 20 => t2_2, 21 => vec3, 22 => const_val) inputs = (t2, t2_2, vec3, const_val) - + arity_map = OrderedDict{Int8,Int}( 1 => 2, 2 => 2, 3 => 2, 4 => 2, 5 => 2, 6 => 2, 7 => 2, # Binary ops 8 => 1, 9 => 1, 10 => 1, 11 => 1, 12 => 1, 13 => 1, # Unary ops part 1 14 => 1, 15 => 1, 16 => 1, 17 => 2, 18 => 1 # Unary ops part 2 + DC ) - + callbacks = Dict{Int8,Any}( Int8(1) => AdditionNode, Int8(2) => SubtractionNode, @@ -138,7 +139,7 @@ using OrderedCollections Int8(17) => DoubleContractionNode, Int8(18) => DeviatoricNode ) - + @testset "Basic Node Operations" begin nodes = OrderedDict{Int8,Any}( Int8(19) => InputSelector(1), @@ -146,89 +147,89 @@ using OrderedCollections Int8(21) => InputSelector(3), Int8(22) => const_val ) - + # Test 1: Addition rek_string = Int8[1, 19, 20] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ t2 + t2_2 - + # Test 2: Subtraction rek_string = Int8[2, 19, 20] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ t2 - t2_2 - + # Test 3: Multiplication with constant rek_string = Int8[3, 19, 22] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ t2 * const_val end - + @testset "Tensor Operations" begin nodes = OrderedDict{Int8,Any}( Int8(19) => InputSelector(1), Int8(20) => InputSelector(2) ) - + # Test 1: Double Contraction rek_string = Int8[17, 19, 20] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ dcontract(t2, t2_2) - + # Test 2: Deviatoric rek_string = Int8[14, 19] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ dev(t2) - + # Test 3: Trace + Symmetric rek_string = Int8[9, 11, 19] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ tr(symmetric(t2)) end - + @testset "Complex Compositions" begin nodes = OrderedDict{Int8,Any}( Int8(19) => InputSelector(1), Int8(20) => InputSelector(2), Int8(22) => const_val ) - + # Test 1: (t2 ⊡ t2_2) * const_val rek_string = Int8[3, 17, 19, 20, 22] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ dcontract(t2, t2_2) * const_val - + # Test 2: dev(symmetric(t2)) + t2_2 rek_string = Int8[1, 14, 11, 19, 20] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ dev(symmetric(t2)) + t2_2 - + # Test 3: tr(t2) * tr(t2_2) rek_string = Int8[3, 9, 19, 9, 20] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ tr(t2) * tr(t2_2) end - + @testset "Error Cases" begin nodes = OrderedDict{Int8,Any}( Int8(19) => InputSelector(1), Int8(21) => InputSelector(3), Int8(22) => const_val ) - + # Test 1: Dimension mismatch (tensor + vector) rek_string = Int8[1, 19, 21] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test_throws DimensionMismatch network(inputs) - + # Test 2: Division by zero data_zero = merge(data_, Dict(22 => 0.0)) rek_string = Int8[4, 19, 22] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test_throws MethodError network(inputs) - + # Test 3: Invalid double contraction rek_string = Int8[17, 19, 21] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test_throws MethodError network(inputs) end - end +end \ No newline at end of file From 03ae430ce854ac6a07aba83d952f355cfb19fbe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sat, 25 Jan 2025 14:02:39 +1100 Subject: [PATCH 36/41] reduce expcetion throwing --- src/Entities.jl | 6 +++--- src/Gep.jl | 3 ++- src/RegressionWrapper.jl | 11 ++++++----- src/TensorOps.jl | 24 +++++++++++++++++++++++- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index 4b03ff1..e867406 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -171,13 +171,13 @@ struct StandardRegressionStrategy{T<:AbstractFloat} <: EvaluationStrategy end struct GenericRegressionStrategy <: EvaluationStrategy - operators::OperatorEnum + operators::Union{OperatorEnum,Nothing} number_of_objectives::Int loss_function::Function secOptimizer::Union{Function,Nothing} break_condition::Union{Function,Nothing} - function GenericRegressionStrategy(operators::OperatorEnum, number_of_objectives::Int, loss_function::Function; + function GenericRegressionStrategy(operators::Union{OperatorEnum,Nothing}, number_of_objectives::Int, loss_function::Function; secOptimizer::Union{Function,Nothing}, break_condition::Union{Function,Nothing}) new(operators, number_of_objectives, loss_function, secOptimizer, break_condition) end @@ -250,7 +250,7 @@ struct Toolbox gen_start_indices = [gene_count + (gene_len * (i - 1)) for i in 1:gene_count] ensure_buffer_size!(head_len, gene_count) new(gene_count, head_len, symbols, gene_connections, headsyms, unary_syms, tailsyms, symbols, - callbacks, nodes, gen_start_indices, gep_probs, unary_prob, fitness_reset, preamble_syms, len_preamble, operators_, compile_djl_datatype) + callbacks, nodes, gen_start_indices, gep_probs, unary_prob, fitness_reset, preamble_syms, len_preamble, operators_, function_complile) end end diff --git a/src/Gep.jl b/src/Gep.jl index 8645705..d21cf3a 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -297,7 +297,8 @@ The evolution process stops when either: same = Atomic{Int}(0) perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) - Threads.@threads for i in eachindex(population) + #Threads.@threads + for i in eachindex(population) if isnan(mean(population[i].fitness)) cache_value = get(fit_cache, population[i].expression_raw, nothing) if isnothing(cache_value) diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index 403b47a..a313d7e 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -153,6 +153,8 @@ using DynamicExpressions using OrderedCollections using LinearAlgebra +const InputSelector = TensorRegUtils.InputSelector + """ FUNCTION_LIB_FORWARD_COMMON::Dict{Symbol,Function} @@ -593,8 +595,8 @@ mutable struct GepTensorRegressor entered_terminal_nums::Vector{<:AbstractFloat}=[0.0, 0.5], gene_connections::Vector{Symbol}=[:+, :-, :*, :/], rnd_count::Int=1, - gene_count::Int=3, - head_len::Int=6, + gene_count::Int=2, + head_len::Int=3, number_of_objectives::Int=1 ) #Creating the feature Nodes -> asuming a data dict pointing to @@ -627,12 +629,11 @@ mutable struct GepTensorRegressor callbacks[cur_idx] = TENSOR_NODES[elem] utilized_symbols[cur_idx] = TENSOR_NODES_ARITY[elem] if elem in gene_connections - push!(gene_connections_) + push!(gene_connections_,cur_idx) end cur_idx += 1 end - toolbox = GepRegression.GepEntities.Toolbox(gene_count, head_len, utilized_symbols, gene_connections_, callbacks, nodes, GENE_COMMON_PROBS; number_of_objectives=number_of_objectives, operators_=nothing, function_complile=compile_to_flux_network) @@ -803,7 +804,7 @@ function fit!(regressor::GepTensorRegressor, epochs::Int, population_size::Int, ) evalStrat = GenericRegressionStrategy( - regressor.operators_, + nothing, length(regressor.toolbox_.fitness_reset[1]), loss_function; secOptimizer=nothing, diff --git a/src/TensorOps.jl b/src/TensorOps.jl index 4f7bcc4..54984c4 100644 --- a/src/TensorOps.jl +++ b/src/TensorOps.jl @@ -193,10 +193,25 @@ end @fastmath return x + y end +@inline function (l::AdditionNode)(x::Union{Tensor,SymmetricTensor}, y::Number) + @fastmath return NaN +end + +@inline function (l::AdditionNode)(x::Number, y::Union{Tensor,SymmetricTensor}) + @fastmath return NaN +end + @inline function (l::SubtractionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) @fastmath return x - y end +@inline function (l::SubtractionNode)(x::Union{Tensor,SymmetricTensor}, y::Number) + @fastmath return NaN +end + +@inline function (l::SubtractionNode)(x::Number, y::Union{Tensor,SymmetricTensor}) + @fastmath return NaN +end @inline function (l::SubtractionNode)(x::Vector, y::Vector) @fastmath return x - y @@ -222,10 +237,17 @@ end @fastmath return dot(x, y) end -@inline function (l::DivisionNode)(x::Union{Tensor,SymmetricTensor}, y::Vector) +@inline function (l::DivisionNode)(x::Union{Tensor,SymmetricTensor,Number}, y::Number) @fastmath return x / y end +@inline function (l::DivisionNode)(x::Number, y::Union{Tensor,SymmetricTensor}) + @fastmath return NaN +end + +@inline function (l::DivisionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) + @fastmath return NaN +end @inline function (l::PowerNode)(x::Union{Tensor,SymmetricTensor,Number}, y::Number) @fastmath return x^y From c350a7d61b3aff6577a42d6b55f8b469bc828538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Sat, 25 Jan 2025 17:54:53 +1100 Subject: [PATCH 37/41] type --- benchmark/benchmark_djl_nn.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmark_djl_nn.jl b/benchmark/benchmark_djl_nn.jl index 23d9e1b..daaca4c 100644 --- a/benchmark/benchmark_djl_nn.jl +++ b/benchmark/benchmark_djl_nn.jl @@ -78,4 +78,4 @@ end #11.703 ms (998979 allocations: 59.49 MiB) -#Conclusion -> 4 times faster than DynamicExpressions.jl for such structures \ No newline at end of file +#Conclusion ≈ 7 times faster than DynamicExpressions.jl for such structures \ No newline at end of file From 5a2c5ff9ac769df16391bc690e9c76d3547798d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Tue, 28 Jan 2025 08:56:43 +1100 Subject: [PATCH 38/41] stop - test --- src/Entities.jl | 45 +++++++++++++++++++++------------------- src/Gep.jl | 4 ++-- src/RegressionWrapper.jl | 17 ++++++++++++--- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/Entities.jl b/src/Entities.jl index e867406..adb5faf 100644 --- a/src/Entities.jl +++ b/src/Entities.jl @@ -74,7 +74,7 @@ module GepEntities export Chromosome, Toolbox, EvaluationStrategy, StandardRegressionStrategy, GenericRegressionStrategy export fitness, set_fitness! -export generate_gene, compile_expression!, generate_chromosome, generate_population +export generate_gene, compile_expression!, generate_chromosome, generate_population export genetic_operations!, replicate, gene_inversion!, gene_mutation!, gene_one_point_cross_over!, gene_two_point_cross_over!, gene_fussion! include("Util.jl") @@ -235,10 +235,10 @@ struct Toolbox function Toolbox(gene_count::Int, head_len::Int, symbols::OrderedDict{Int8,Int8}, gene_connections::Vector{Int8}, callbacks::Dict, nodes::OrderedDict, gep_probs::Dict{String,AbstractFloat}; - unary_prob::Real=0.1, preamble_syms=Int8[], + unary_prob::Real=0.1, preamble_syms=Int8[], number_of_objectives::Int=1, operators_::Union{OperatorEnum,Nothing}=nothing, function_complile::Function=compile_djl_datatype) - - fitness_reset= ( + + fitness_reset = ( ntuple(_ -> Inf, number_of_objectives), ntuple(_ -> NaN, number_of_objectives) ) @@ -246,7 +246,7 @@ struct Toolbox headsyms = [key for (key, arity) in symbols if arity == 2] unary_syms = [key for (key, arity) in symbols if arity == 1] tailsyms = [key for (key, arity) in symbols if arity < 1 && !(key in preamble_syms)] - len_preamble = length(preamble_syms) + len_preamble = length(preamble_syms) gen_start_indices = [gene_count + (gene_len * (i - 1)) for i in 1:gene_count] ensure_buffer_size!(head_len, gene_count) new(gene_count, head_len, symbols, gene_connections, headsyms, unary_syms, tailsyms, symbols, @@ -327,13 +327,13 @@ Fitness value or tuple try expression = _karva_raw(chromosome) expression_tree = chromosome.toolbox.compile_function_(expression, chromosome.toolbox.symbols, chromosome.toolbox.callbacks, - chromosome.toolbox.nodes, max(chromosome.toolbox.len_preamble,1)) + chromosome.toolbox.nodes, max(chromosome.toolbox.len_preamble, 1)) chromosome.compiled_function = expression_tree chromosome.expression_raw = expression chromosome.fitness = chromosome.toolbox.fitness_reset[2] chromosome.compiled = true catch e - @error "something went wrong" exception=(e,catch_backtrace()) + @error "something went wrong" exception = (e, catch_backtrace()) chromosome.fitness = chromosome.toolbox.fitness_reset[1] end end @@ -416,7 +416,7 @@ Vector{Int8} representing the K-expression of the chromosome gene_count = chromosome.toolbox.gene_count connectionsym = @view chromosome.genes[1:gene_count-1] - genes = chromosome.genes[gene_count:end] + genes = chromosome.genes[gene_count:end] arity_gene_ = map(x -> chromosome.toolbox.arrity_by_id[x], genes) rolled_indices = Vector{Any}(undef, div(length(arity_gene_), gene_len) + 1) @@ -449,9 +449,10 @@ Generate a single gene for GEP. # Returns Vector{Int8} representing gene """ -@inline function generate_gene(headsyms::Vector{Int8}, tailsyms::Vector{Int8}, headlen::Int; unarys::Vector{Int8}=[], unary_prob::Real=0.2) +@inline function generate_gene(headsyms::Vector{Int8}, tailsyms::Vector{Int8}, headlen::Int; + unarys::Vector{Int8}=[], unary_prob::Real=0.2, tensor_prob::Real=0.2) if !isempty(unarys) && rand() < unary_prob - heads = vcat(headsyms,tailsyms) + heads = vcat(headsyms, tailsyms) push!(heads, rand(unarys)) else heads = headsyms @@ -462,6 +463,8 @@ Vector{Int8} representing gene return vcat(head, tail) end + + """ generate_chromosome(toolbox::Toolbox) @@ -493,7 +496,7 @@ Vector of Chromosomes """ @inline function generate_population(number::Int, toolbox::Toolbox) population = Vector{Chromosome}(undef, number) - + Threads.@threads for i in 1:number @inbounds population[i] = generate_chromosome(toolbox) end @@ -523,7 +526,7 @@ end buffer.alpha_operator[1:len_a] .= zeros(Int8) buffer.beta_operator[1:len_a] .= zeros(Int8) - + head_len = toolbox.head_len gene_len = head_len * 2 + 1 @@ -581,7 +584,7 @@ end @inline function gene_dominant_fusion!(chromosome1::Chromosome, chromosome2::Chromosome, pb::Real=0.2) buffer = THREAD_BUFFERS[Threads.threadid()] - len_a =length(chromosome1.genes) + len_a = length(chromosome1.genes) create_operator_masks(chromosome1.genes, chromosome2.genes, pb) @inbounds @simd for i in eachindex(chromosome1.genes) @@ -590,12 +593,12 @@ end end chromosome1.genes .= @view buffer.child_1_genes[1:len_a] - chromosome2.genes .= @view buffer.child_2_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gen_rezessiv!(chromosome1::Chromosome, chromosome2::Chromosome, pb::Real=0.2) buffer = THREAD_BUFFERS[Threads.threadid()] - len_a =length(chromosome1.genes) + len_a = length(chromosome1.genes) create_operator_masks(chromosome1.genes, chromosome2.genes, pb) @inbounds @simd for i in eachindex(chromosome1.genes) @@ -604,7 +607,7 @@ end end chromosome1.genes .= @view buffer.child_1_genes[1:len_a] - chromosome2.genes .= @view buffer.child_2_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gene_fussion!(chromosome1::Chromosome, chromosome2::Chromosome, pb::Real=0.2) @@ -618,7 +621,7 @@ end end chromosome1.genes .= @view buffer.child_1_genes[1:len_a] - chromosome2.genes .= @view buffer.child_2_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gene_one_point_cross_over!(chromosome1::Chromosome, chromosome2::Chromosome) @@ -632,7 +635,7 @@ end end chromosome1.genes .= @view buffer.child_1_genes[1:len_a] - chromosome2.genes .= @view buffer.child_2_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gene_two_point_cross_over!(chromosome1::Chromosome, chromosome2::Chromosome) @@ -646,7 +649,7 @@ end end chromosome1.genes .= @view buffer.child_1_genes[1:len_a] - chromosome2.genes .= @view buffer.child_2_genes[1:len_a] + chromosome2.genes .= @view buffer.child_2_genes[1:len_a] end @inline function gene_mutation!(chromosome1::Chromosome, pb::Real=0.25) @@ -656,7 +659,7 @@ end @inbounds @simd for i in eachindex(chromosome1.genes) chromosome1.genes[i] = buffer.alpha_operator[i] == 1 ? buffer.child_1_genes[i] : chromosome1.genes[i] - end + end end @inline function gene_inversion!(chromosome1::Chromosome) @@ -678,7 +681,7 @@ end end @inline function reverse_insertion_tail!(chromosome::Chromosome) - start_1 = rand(chromosome.toolbox.gen_start_indices)+chromosome.toolbox.head_len+1 + start_1 = rand(chromosome.toolbox.gen_start_indices) + chromosome.toolbox.head_len + 1 rolled_array = circshift(chromosome.genes[start_1:start_1+chromosome.toolbox.head_len-1], rand(1:chromosome.toolbox.head_len-1)) chromosome.genes[start_1:start_1+chromosome.toolbox.head_len-1] = rolled_array end diff --git a/src/Gep.jl b/src/Gep.jl index d21cf3a..33d23a7 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -297,8 +297,8 @@ The evolution process stops when either: same = Atomic{Int}(0) perform_correction_callback!(population, epoch, correction_epochs, correction_amount, correction_callback) - #Threads.@threads - for i in eachindex(population) + + Threads.@threads for i in eachindex(population) if isnan(mean(population[i].fitness)) cache_value = get(fit_cache, population[i].expression_raw, nothing) if isnothing(cache_value) diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index a313d7e..5ec8db9 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -562,7 +562,8 @@ A Gene Expression Programming (GEP) regressor that evolves higher order mathemat Create a new GEP tensor regressor with specified number of input features. # Arguments -- `feature_amount::Int`: Number of input features to use in the regression +- `scalar_feature_amount::Int`: Number of input features representing scalar quantities +- `higher_dim_feature_amount::Int`: Number of input features representing hihger quantities # Keyword Arguments - `entered_non_terminals::Vector{Symbol}=[:+, :-, :*, :/]`: Available mathematical operators @@ -584,13 +585,15 @@ multiple objectives. - Uses TENSOR_NODES_ARITY for operator arity - Compiles expressions to Flux networks via compile_to_flux_network """ +#TODO => adapt probs for occurance of ! mutable struct GepTensorRegressor toolbox_::Toolbox best_models_::Union{Nothing,Vector{Chromosome}} fitness_history_::Any - function GepTensorRegressor(feature_amount::Int; + function GepTensorRegressor(scalar_feature_amount::Int; + higher_dim_feature_amount::Int=0, entered_non_terminals::Vector{Symbol}=[:+, :-, :*, :/], entered_terminal_nums::Vector{<:AbstractFloat}=[0.0, 0.5], gene_connections::Vector{Symbol}=[:+, :-, :*, :/], @@ -605,7 +608,15 @@ mutable struct GepTensorRegressor utilized_symbols = SymbolDict() callbacks = Dict{Int8,Any}() gene_connections_ = Int8[] - for _ in 1:feature_amount + tensor_syms_idx = Int8[] + tensor_function_idx = Int8[] + for _ in 1:scalar_feature_amount + nodes[cur_idx] = InputSelector(cur_idx) + utilized_symbols[cur_idx] = Int8(0) + cur_idx += 1 + end + + for _ in 1:higher_dim_feature_amount nodes[cur_idx] = InputSelector(cur_idx) utilized_symbols[cur_idx] = Int8(0) cur_idx += 1 From ea95c07c157fef9bd08b1f250b05e64f466cae13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Wed, 29 Jan 2025 17:14:11 +1100 Subject: [PATCH 39/41] function save --- Project.toml | 2 + src/Gep.jl | 1 + src/RegressionWrapper.jl | 8 +- src/TensorOps.jl | 434 +++++++++++++++++---------------------- test/Main_min_bench.jl | 4 +- test/tensor_ops_test.jl | 26 +-- 6 files changed, 198 insertions(+), 277 deletions(-) diff --git a/Project.toml b/Project.toml index 2980c65..a24d2a0 100644 --- a/Project.toml +++ b/Project.toml @@ -22,7 +22,9 @@ Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" LoopVectorization = "bdcacae8-1622-11e9-2a5c-532679323890" Optim = "429524aa-4258-5aef-a3af-852621145aeb" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" diff --git a/src/Gep.jl b/src/Gep.jl index 33d23a7..198fb14 100644 --- a/src/Gep.jl +++ b/src/Gep.jl @@ -302,6 +302,7 @@ The evolution process stops when either: if isnan(mean(population[i].fitness)) cache_value = get(fit_cache, population[i].expression_raw, nothing) if isnothing(cache_value) + population[i].fitness = compute_fitness(population[i], evalStrategy) lock(cache_lock) fit_cache[population[i].expression_raw] = population[i].fitness diff --git a/src/RegressionWrapper.jl b/src/RegressionWrapper.jl index 5ec8db9..219b0be 100644 --- a/src/RegressionWrapper.jl +++ b/src/RegressionWrapper.jl @@ -594,10 +594,10 @@ mutable struct GepTensorRegressor function GepTensorRegressor(scalar_feature_amount::Int; higher_dim_feature_amount::Int=0, - entered_non_terminals::Vector{Symbol}=[:+, :-, :*, :/], - entered_terminal_nums::Vector{<:AbstractFloat}=[0.0, 0.5], - gene_connections::Vector{Symbol}=[:+, :-, :*, :/], - rnd_count::Int=1, + entered_non_terminals::Vector{Symbol}=[:+, :-, :*], + entered_terminal_nums::Vector{<:AbstractFloat}=Float64[], + gene_connections::Vector{Symbol}=[:+, :*], + rnd_count::Int=0, gene_count::Int=2, head_len::Int=3, number_of_objectives::Int=1 diff --git a/src/TensorOps.jl b/src/TensorOps.jl index 54984c4..5676619 100644 --- a/src/TensorOps.jl +++ b/src/TensorOps.jl @@ -1,320 +1,296 @@ module TensorRegUtils -using Flux, LinearAlgebra, OrderedCollections, ChainRulesCore +using Flux, LinearAlgebra, OrderedCollections, ChainRulesCore, Tensors, PrecompileTools -using Tensors -# Types -export OperationNode -export InputSelector -export AdditionNode, SubtractionNode, MultiplicationNode, DivisionNode, PowerNode -export MinNode, MaxNode, InversionNode -export TraceNode, DeterminantNode, SymmetricNode, SkewNode -export VolumetricNode, DeviatricNode, TdotNode, DottNode -export DoubleContractionNode, DeviatoricNode -export ConstantNode, UnaryNode +struct ThreadBuffer + vector::Vector{Union{Number,Tensor,SymmetricTensor}} +end -export compile_to_flux_network +const THREAD_BUFFERS = Vector{ThreadBuffer}(undef, Threads.nthreads()) -export TENSOR_NODES, TENSOR_NODES_ARITY +function __init__() + for i in 1:Threads.nthreads() + THREAD_BUFFERS[i] = ThreadBuffer(Vector{Union{Number,Tensor,SymmetricTensor}}(undef, 0)) + end +end +@inline function get_thread_buffer(n::Integer) + buffer = THREAD_BUFFERS[Threads.threadid()].vector + if length(buffer) < n + resize!(buffer, n) + end + buffer +end -abstract type OperationNode end -struct InputSelector - idx::Int -end +# Abstract base type with parametric types for improved type stability +abstract type AbstractOperationNode{T} end -@inline function (l::InputSelector)(x::Tuple) - x[l.idx] +# Input selector with strict typing +struct InputSelector{T<:Integer} + idx::T end -struct AdditionNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - AdditionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::InputSelector{T})(x::Tuple) where T + @inbounds x[l.idx] end -struct SubtractionNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - SubtractionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::InputSelector{T})(x::Any) where T + @inbounds x end -struct MultiplicationNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - MultiplicationNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +# Macro to generate specialized operation nodes with functors +macro generate_operation_node(name) + return quote + struct $(esc(name)){T<:Union{Nothing,Chain}} <: AbstractOperationNode{T} + chain::T + $(esc(name))(chain=nothing) = new{typeof(chain)}(chain) + end + Flux.Functors.@functor $(esc(name)) + end end -struct DivisionNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - DivisionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) -end +# Generate concrete operation nodes +@generate_operation_node AdditionNode +@generate_operation_node SubtractionNode +@generate_operation_node MultiplicationNode +@generate_operation_node DivisionNode +@generate_operation_node PowerNode +@generate_operation_node MinNode +@generate_operation_node MaxNode +@generate_operation_node InversionNode +@generate_operation_node TraceNode +@generate_operation_node DeterminantNode +@generate_operation_node SymmetricNode +@generate_operation_node SkewNode +@generate_operation_node VolumetricNode +@generate_operation_node DeviatricNode +@generate_operation_node TdotNode +@generate_operation_node DottNode +@generate_operation_node DoubleContractionNode +@generate_operation_node DeviatoricNode -struct PowerNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - PowerNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +# Specialized nodes with their functors +struct ConstantNode{T<:Number, C<:Union{Nothing,Chain}} <: AbstractOperationNode{C} + value::T + chain::C + ConstantNode(value::T, chain=nothing) where T<:Number = new{T,typeof(chain)}(value, chain) end +Flux.Functors.@functor ConstantNode -struct MinNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - MinNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +struct UnaryNode{F<:Function, C<:Union{Nothing,Chain}} <: AbstractOperationNode{C} + operation::F + chain::C + UnaryNode(operation::F, chain=nothing) where F<:Function = new{F,typeof(chain)}(operation, chain) end +Flux.Functors.@functor UnaryNode -struct MaxNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - MaxNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +# Operation implementations +@inline function (l::AdditionNode{T})(x::Union{Tensor,SymmetricTensor}, + y::Union{Tensor,SymmetricTensor}) where {T} + @fastmath (x + y)::Union{Tensor,SymmetricTensor} end -struct InversionNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - InversionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::AdditionNode{T})(x::Number, y::Number) where T + @fastmath (x + y)::Number end -struct TraceNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - TraceNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) -end -struct DeterminantNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - DeterminantNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::AdditionNode{T})(x::Any, y::Any) where T + Inf::Number end -struct SymmetricNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - SymmetricNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::MultiplicationNode{T})(x::Any, y::Any) where T + Inf::Number end -struct SkewNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - SkewNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::SubtractionNode{T})(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) where T + @fastmath (x - y)::Union{Tensor,SymmetricTensor} end -struct VolumetricNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - VolumetricNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) -end -struct DeviatricNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - DeviatricNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::SubtractionNode{T})(x::Number, y::Number) where T + @fastmath (x - y)::Number end -struct TdotNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - TdotNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::SubtractionNode{T})(x::Any, y::Any) where T + Inf::Number end -struct DottNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - DottNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::MultiplicationNode{T})(x::Number, y::Number) where T + @fastmath (x * y)::Number end -struct DoubleContractionNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - DoubleContractionNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::MultiplicationNode{T})(x::Union{Tensor,SymmetricTensor}, y::Number) where T + @fastmath (x * y)::Union{Tensor,SymmetricTensor} end -struct DeviatoricNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - DeviatoricNode(chain=Chain(); use_cuda::Bool=false) = new(chain, use_cuda) +@inline function (l::MultiplicationNode{T})(x::Number, y::Union{Tensor,SymmetricTensor}) where T + @fastmath (x * y)::Union{Tensor,SymmetricTensor} end -struct ConstantNode <: OperationNode - value::Number - chain::Union{Nothing,Chain} - use_cuda::Bool - ConstantNode(value::Number, chain=Chain(); use_cuda::Bool=false) = new(value, chain, use_cuda) -end -struct UnaryNode <: OperationNode - operation::Function - chain::Union{Nothing,Chain} - use_cuda::Bool - UnaryNode(operation::Function, chain=Chain(); use_cuda::Bool=false) = - new(operation, chain, use_cuda) +@inline function (l::MultiplicationNode{T})(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) where T + @fastmath dot(x, y)::Union{Tensor,SymmetricTensor} end +@inline function (l::DivisionNode{T})(x::Union{Tensor,SymmetricTensor,Number}, y::Number) where T + @fastmath (x / y)::Union{Tensor,SymmetricTensor,Number} +end -const NODES = [ - AdditionNode, - SubtractionNode, - MultiplicationNode, - DivisionNode, - PowerNode, - MinNode, - MaxNode, - InversionNode, - TraceNode, - DeterminantNode, - SymmetricNode, - SkewNode, - VolumetricNode, - DeviatricNode, - TdotNode, - DottNode, - DoubleContractionNode, - DeviatoricNode, - ConstantNode, - UnaryNode -] +@inline function (l::DivisionNode{T})(x::Any, y::Any) where T + Inf::Number +end -for T in NODES - @eval Flux.Functors.@functor $T +@inline function (l::PowerNode{T})(x::Union{Tensor,SymmetricTensor,Number}, y::Number) where T + @fastmath (x^y)::Union{Tensor,SymmetricTensor} end -@inline function (l::AdditionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) - @fastmath return x + y +@inline function (l::DoubleContractionNode{T})(x, y) where T + @fastmath dcontract(x, y) end -@inline function (l::AdditionNode)(x::Vector, y::Vector) - @fastmath return x + y +@inline function (l::DeviatoricNode{T})(x) where T + @fastmath dev(x) end -@inline function (l::AdditionNode)(x::Number, y::Number) - @fastmath return x + y +@inline function (l::MinNode{T})(x, y) where T + @fastmath min(x, y) end -@inline function (l::AdditionNode)(x::Union{Tensor,SymmetricTensor}, y::Number) - @fastmath return NaN +@inline function (l::MaxNode{T})(x, y) where T + @fastmath max(x, y) end -@inline function (l::AdditionNode)(x::Number, y::Union{Tensor,SymmetricTensor}) - @fastmath return NaN +@inline function (l::InversionNode{T})(x) where T + @fastmath inv(x) end -@inline function (l::SubtractionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) - @fastmath return x - y +@inline function (l::TraceNode{T})(x) where T + @fastmath tr(x) end -@inline function (l::SubtractionNode)(x::Union{Tensor,SymmetricTensor}, y::Number) - @fastmath return NaN +@inline function (l::DeterminantNode{T})(x) where T + @fastmath det(x)::Number end -@inline function (l::SubtractionNode)(x::Number, y::Union{Tensor,SymmetricTensor}) - @fastmath return NaN +@inline function (l::SymmetricNode{T})(x::Union{Tensor,SymmetricTensor}) where T + @fastmath symmetric(x)::Union{Tensor,SymmetricTensor} end -@inline function (l::SubtractionNode)(x::Vector, y::Vector) - @fastmath return x - y +@inline function (l::SkewNode{T})(x::Union{Tensor,SymmetricTensor}) where T + @fastmath skew(x)::Union{Tensor,SymmetricTensor} end -@inline function (l::SubtractionNode)(x::Number, y::Number) - @fastmath return x - y +@inline function (l::VolumetricNode{T})(x) where T + @fastmath vol(x) end -@inline function (l::MultiplicationNode)(x::Number, y::Number) - @fastmath return y * x +@inline function (l::DeviatricNode{T})(x) where T + @fastmath dev(x) end -@inline function (l::MultiplicationNode)(x::Union{Tensor,SymmetricTensor}, y::Number) - @fastmath return y * x +@inline function (l::TdotNode{T})(x) where T + @fastmath tdot(x) end -@inline function (l::MultiplicationNode)(x::Number, y::Union{Tensor,SymmetricTensor}) - @fastmath return y * x +@inline function (l::DottNode{T})(x) where T + @fastmath dott(x) end -@inline function (l::MultiplicationNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) - @fastmath return dot(x, y) +@inline function (l::ConstantNode{V,T})(x) where {V,T} + l.value end -@inline function (l::DivisionNode)(x::Union{Tensor,SymmetricTensor,Number}, y::Number) - @fastmath return x / y +@inline function (l::UnaryNode{F,T})(x) where {F,T} + @fastmath l.operation.(x) end -@inline function (l::DivisionNode)(x::Number, y::Union{Tensor,SymmetricTensor}) - @fastmath return NaN + +@inline function (l::AdditionNode{T})(x::AbstractVector, y::AbstractVector) where T + map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::DivisionNode)(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) - @fastmath return NaN +@inline function (l::AdditionNode{T})(x::AbstractVector{Number}, y::AbstractVector{Number}) where T + (x .+ y)::AbstractVector{Number} end -@inline function (l::PowerNode)(x::Union{Tensor,SymmetricTensor,Number}, y::Number) - @fastmath return x^y +@inline function (l::SubtractionNode{T})(x::AbstractVector, y::AbstractVector) where T + map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::DoubleContractionNode)(x, y) - @fastmath return dcontract(x, y) +@inline function (l::MultiplicationNode{T})(x::AbstractVector, y::AbstractVector) where T + map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::DeviatoricNode)(x) - @fastmath return dev(x) +@inline function (l::DivisionNode{T})(x::AbstractVector, y::Number) where T + map(a -> l(a, y), x)::AbstractVector end +@inline function (l::DivisionNode{T})(x::AbstractVector, y::AbstractVector) where T + map((a, b) -> l(a, b), x, y)::AbstractVector +end -@inline function (l::MinNode)(x, y) - @fastmath return min(x, y) +@inline function (l::PowerNode{T})(x::AbstractVector, y::Number) where T + map(a -> l(a, y), x)::AbstractVector end -@inline function (l::MaxNode)(x, y) - @fastmath return max(x, y) +@inline function (l::PowerNode{T})(x::AbstractVector, y::AbstractVector) where T + map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::InversionNode)(x) - @fastmath return inv(x) +@inline function (l::MinNode{T})(x::AbstractVector, y::AbstractVector) where T + map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::TraceNode)(x) - @fastmath return tr(x) +@inline function (l::MaxNode{T})(x::AbstractVector, y::AbstractVector) where T + map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::DeterminantNode)(x) - @fastmath return det(x) +@inline function (l::TraceNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end -@inline function (l::SymmetricNode)(x) - @fastmath return symmetric(x) +@inline function (l::DeterminantNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end -@inline function (l::SkewNode)(x) - @fastmath return skew(x) +@inline function (l::SymmetricNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end -@inline function (l::VolumetricNode)(x) - @fastmath return vol(x) +@inline function (l::SkewNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end -@inline function (l::DeviatricNode)(x) - @fastmath return dev(x) +@inline function (l::VolumetricNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end -@inline function (l::TdotNode)(x) - @fastmath return tdot(x) +@inline function (l::DeviatricNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end -@inline function (l::DottNode)(x) - @fastmath return dott(x) +@inline function (l::InversionNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end -@inline function (l::ConstantNode)(x) - return l.value +@inline function (l::TdotNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end +@inline function (l::DottNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector +end -@inline function (l::UnaryNode)(x) - @fastmath return l.operation.(x) +@inline function (l::DeviatoricNode{T})(x::AbstractVector) where T + map(l, x)::AbstractVector end + + function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, callbacks::Dict, nodes::OrderedDict, pre_len::Int) stack = [] for elem in reverse(rek_string) @@ -341,7 +317,7 @@ function compile_to_flux_network(rek_string::Vector, arity_map::OrderedDict, cal return Chain(pop!(stack)) end - +# Constant mappings const TENSOR_NODES = Dict{Symbol,Type}( :+ => AdditionNode, :- => SubtractionNode, @@ -364,57 +340,23 @@ const TENSOR_NODES = Dict{Symbol,Type}( ) const TENSOR_NODES_ARITY = Dict{Symbol,Int8}( - :+ => 2, - :- => 2, - :* => 2, - :/ => 2, - :^ => 2, - :min => 2, - :max => 2, - :inv => 1, - :tr => 1, - :det => 1, - :symmetric => 1, - :skew => 1, - :vol => 1, - :dev => 1, - :tdot => 1, - :dott => 1, - :dcontract => 2, - :deviator => 1 + :+ => 2, :- => 2, :* => 2, :/ => 2, :^ => 2, + :min => 2, :max => 2, + :inv => 1, :tr => 1, :det => 1, + :symmetric => 1, :skew => 1, :vol => 1, :dev => 1, + :tdot => 1, :dott => 1, :dcontract => 2, :deviator => 1 ) +# Exports +export OperationNode, InputSelector +export AdditionNode, SubtractionNode, MultiplicationNode, DivisionNode, PowerNode +export MinNode, MaxNode, InversionNode +export TraceNode, DeterminantNode, SymmetricNode, SkewNode +export VolumetricNode, DeviatricNode, TdotNode, DottNode +export DoubleContractionNode, DeviatoricNode +export ConstantNode, UnaryNode +export compile_to_flux_network +export TENSOR_NODES, TENSOR_NODES_ARITY -""" - -Later on cuda Module - -@inline function ensure_cuda(x, use_cuda::Bool) - if use_cuda && !isa(x, CuArray) - cu(x) - elseif !use_cuda && isa(x, CuArray) - cpu(x) - else - return x - end -end - -Example Method usage: - -struct DottNode <: OperationNode - chain::Union{Nothing,Chain} - use_cuda::Bool - DottNode(chain=nothing; use_cuda::Bool=false) = new(chain, use_cuda) -end - -notes: -if rek comes to a constant: ConstantNode(elem, nothing, use_cuda=use_cuda) -if rek comes to an x -> InputSelector(i) -if unary -> initializes Arity1Node(nothing, use_cuda=use_cuda) -if binary -> initaliszed Arity2Node(nothing, use_cuda=use_cuda) - -@assert sp == 1 "Invalid expression: stack error" - -""" end \ No newline at end of file diff --git a/test/Main_min_bench.jl b/test/Main_min_bench.jl index 3110154..478c2e1 100644 --- a/test/Main_min_bench.jl +++ b/test/Main_min_bench.jl @@ -8,8 +8,8 @@ using BenchmarkTools Random.seed!(1) #Define the iterations for the algorithm and the population size -epochs = 1000 -population_size = 1500 +epochs = 10 +population_size = 20 #Number of features which needs to be inserted number_features = 2 diff --git a/test/tensor_ops_test.jl b/test/tensor_ops_test.jl index fc8de25..5c48ab0 100644 --- a/test/tensor_ops_test.jl +++ b/test/tensor_ops_test.jl @@ -91,7 +91,7 @@ using Flux # (t2 * vec3) + const_val - should fail rek_string = Int8[1, 2, 5, 6, 7] network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - @test_throws MethodError network(inputs) + @test !isfinite(network(inputs)) # tr(t2) * const_val + tr(t2) rek_string = Int8[1, 2, 4, 5, 7, 4, 5] @@ -208,28 +208,4 @@ end network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) @test network(inputs) ≈ tr(t2) * tr(t2_2) end - - @testset "Error Cases" begin - nodes = OrderedDict{Int8,Any}( - Int8(19) => InputSelector(1), - Int8(21) => InputSelector(3), - Int8(22) => const_val - ) - - # Test 1: Dimension mismatch (tensor + vector) - rek_string = Int8[1, 19, 21] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - @test_throws DimensionMismatch network(inputs) - - # Test 2: Division by zero - data_zero = merge(data_, Dict(22 => 0.0)) - rek_string = Int8[4, 19, 22] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - @test_throws MethodError network(inputs) - - # Test 3: Invalid double contraction - rek_string = Int8[17, 19, 21] - network = compile_to_flux_network(rek_string, arity_map, callbacks, nodes, 0) - @test_throws MethodError network(inputs) - end end \ No newline at end of file From 0b9e7a91d16fbff64af5aef0bab55fdc7d11ca79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Thu, 30 Jan 2025 21:37:53 +1100 Subject: [PATCH 40/41] adding precompile test green + create tensor --- src/TensorOps.jl | 155 ++++++++++++++++++++++++++++++----------------- 1 file changed, 101 insertions(+), 54 deletions(-) diff --git a/src/TensorOps.jl b/src/TensorOps.jl index 5676619..d79eb8b 100644 --- a/src/TensorOps.jl +++ b/src/TensorOps.jl @@ -32,11 +32,11 @@ struct InputSelector{T<:Integer} idx::T end -@inline function (l::InputSelector{T})(x::Tuple) where T +@inline function (l::InputSelector{T})(x::Tuple) where {T} @inbounds x[l.idx] end -@inline function (l::InputSelector{T})(x::Any) where T +@inline function (l::InputSelector{T})(x::Any) where {T} @inbounds x end @@ -72,131 +72,131 @@ end @generate_operation_node DeviatoricNode # Specialized nodes with their functors -struct ConstantNode{T<:Number, C<:Union{Nothing,Chain}} <: AbstractOperationNode{C} +struct ConstantNode{T<:Number,C<:Union{Nothing,Chain}} <: AbstractOperationNode{C} value::T chain::C - ConstantNode(value::T, chain=nothing) where T<:Number = new{T,typeof(chain)}(value, chain) + ConstantNode(value::T, chain=nothing) where {T<:Number} = new{T,typeof(chain)}(value, chain) end Flux.Functors.@functor ConstantNode -struct UnaryNode{F<:Function, C<:Union{Nothing,Chain}} <: AbstractOperationNode{C} +struct UnaryNode{F<:Function,C<:Union{Nothing,Chain}} <: AbstractOperationNode{C} operation::F chain::C - UnaryNode(operation::F, chain=nothing) where F<:Function = new{F,typeof(chain)}(operation, chain) + UnaryNode(operation::F, chain=nothing) where {F<:Function} = new{F,typeof(chain)}(operation, chain) end Flux.Functors.@functor UnaryNode # Operation implementations -@inline function (l::AdditionNode{T})(x::Union{Tensor,SymmetricTensor}, - y::Union{Tensor,SymmetricTensor}) where {T} +@inline function (l::AdditionNode{T})(x::Union{Tensor,SymmetricTensor}, + y::Union{Tensor,SymmetricTensor}) where {T} @fastmath (x + y)::Union{Tensor,SymmetricTensor} end -@inline function (l::AdditionNode{T})(x::Number, y::Number) where T +@inline function (l::AdditionNode{T})(x::Number, y::Number) where {T} @fastmath (x + y)::Number end -@inline function (l::AdditionNode{T})(x::Any, y::Any) where T +@inline function (l::AdditionNode{T})(x::Any, y::Any) where {T} Inf::Number end -@inline function (l::MultiplicationNode{T})(x::Any, y::Any) where T +@inline function (l::MultiplicationNode{T})(x::Any, y::Any) where {T} Inf::Number end -@inline function (l::SubtractionNode{T})(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) where T +@inline function (l::SubtractionNode{T})(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) where {T} @fastmath (x - y)::Union{Tensor,SymmetricTensor} end -@inline function (l::SubtractionNode{T})(x::Number, y::Number) where T +@inline function (l::SubtractionNode{T})(x::Number, y::Number) where {T} @fastmath (x - y)::Number end -@inline function (l::SubtractionNode{T})(x::Any, y::Any) where T +@inline function (l::SubtractionNode{T})(x::Any, y::Any) where {T} Inf::Number end -@inline function (l::MultiplicationNode{T})(x::Number, y::Number) where T +@inline function (l::MultiplicationNode{T})(x::Number, y::Number) where {T} @fastmath (x * y)::Number end -@inline function (l::MultiplicationNode{T})(x::Union{Tensor,SymmetricTensor}, y::Number) where T +@inline function (l::MultiplicationNode{T})(x::Union{Tensor,SymmetricTensor}, y::Number) where {T} @fastmath (x * y)::Union{Tensor,SymmetricTensor} end -@inline function (l::MultiplicationNode{T})(x::Number, y::Union{Tensor,SymmetricTensor}) where T +@inline function (l::MultiplicationNode{T})(x::Number, y::Union{Tensor,SymmetricTensor}) where {T} @fastmath (x * y)::Union{Tensor,SymmetricTensor} end -@inline function (l::MultiplicationNode{T})(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) where T +@inline function (l::MultiplicationNode{T})(x::Union{Tensor,SymmetricTensor}, y::Union{Tensor,SymmetricTensor}) where {T} @fastmath dot(x, y)::Union{Tensor,SymmetricTensor} end -@inline function (l::DivisionNode{T})(x::Union{Tensor,SymmetricTensor,Number}, y::Number) where T +@inline function (l::DivisionNode{T})(x::Union{Tensor,SymmetricTensor,Number}, y::Number) where {T} @fastmath (x / y)::Union{Tensor,SymmetricTensor,Number} end -@inline function (l::DivisionNode{T})(x::Any, y::Any) where T +@inline function (l::DivisionNode{T})(x::Any, y::Any) where {T} Inf::Number end -@inline function (l::PowerNode{T})(x::Union{Tensor,SymmetricTensor,Number}, y::Number) where T +@inline function (l::PowerNode{T})(x::Union{Tensor,SymmetricTensor,Number}, y::Number) where {T} @fastmath (x^y)::Union{Tensor,SymmetricTensor} end -@inline function (l::DoubleContractionNode{T})(x, y) where T +@inline function (l::DoubleContractionNode{T})(x, y) where {T} @fastmath dcontract(x, y) end -@inline function (l::DeviatoricNode{T})(x) where T +@inline function (l::DeviatoricNode{T})(x) where {T} @fastmath dev(x) end -@inline function (l::MinNode{T})(x, y) where T +@inline function (l::MinNode{T})(x, y) where {T} @fastmath min(x, y) end -@inline function (l::MaxNode{T})(x, y) where T +@inline function (l::MaxNode{T})(x, y) where {T} @fastmath max(x, y) end -@inline function (l::InversionNode{T})(x) where T +@inline function (l::InversionNode{T})(x) where {T} @fastmath inv(x) end -@inline function (l::TraceNode{T})(x) where T +@inline function (l::TraceNode{T})(x) where {T} @fastmath tr(x) end -@inline function (l::DeterminantNode{T})(x) where T +@inline function (l::DeterminantNode{T})(x) where {T} @fastmath det(x)::Number end -@inline function (l::SymmetricNode{T})(x::Union{Tensor,SymmetricTensor}) where T +@inline function (l::SymmetricNode{T})(x::Union{Tensor,SymmetricTensor}) where {T} @fastmath symmetric(x)::Union{Tensor,SymmetricTensor} end -@inline function (l::SkewNode{T})(x::Union{Tensor,SymmetricTensor}) where T +@inline function (l::SkewNode{T})(x::Union{Tensor,SymmetricTensor}) where {T} @fastmath skew(x)::Union{Tensor,SymmetricTensor} end -@inline function (l::VolumetricNode{T})(x) where T +@inline function (l::VolumetricNode{T})(x) where {T} @fastmath vol(x) end -@inline function (l::DeviatricNode{T})(x) where T +@inline function (l::DeviatricNode{T})(x) where {T} @fastmath dev(x) end -@inline function (l::TdotNode{T})(x) where T +@inline function (l::TdotNode{T})(x) where {T} @fastmath tdot(x) end -@inline function (l::DottNode{T})(x) where T +@inline function (l::DottNode{T})(x) where {T} @fastmath dott(x) end @@ -209,83 +209,83 @@ end end -@inline function (l::AdditionNode{T})(x::AbstractVector, y::AbstractVector) where T +@inline function (l::AdditionNode{T})(x::AbstractVector, y::AbstractVector) where {T} map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::AdditionNode{T})(x::AbstractVector{Number}, y::AbstractVector{Number}) where T +@inline function (l::AdditionNode{T})(x::AbstractVector{Number}, y::AbstractVector{Number}) where {T} (x .+ y)::AbstractVector{Number} end -@inline function (l::SubtractionNode{T})(x::AbstractVector, y::AbstractVector) where T +@inline function (l::SubtractionNode{T})(x::AbstractVector, y::AbstractVector) where {T} map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::MultiplicationNode{T})(x::AbstractVector, y::AbstractVector) where T +@inline function (l::MultiplicationNode{T})(x::AbstractVector, y::AbstractVector) where {T} map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::DivisionNode{T})(x::AbstractVector, y::Number) where T +@inline function (l::DivisionNode{T})(x::AbstractVector, y::Number) where {T} map(a -> l(a, y), x)::AbstractVector end -@inline function (l::DivisionNode{T})(x::AbstractVector, y::AbstractVector) where T +@inline function (l::DivisionNode{T})(x::AbstractVector, y::AbstractVector) where {T} map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::PowerNode{T})(x::AbstractVector, y::Number) where T +@inline function (l::PowerNode{T})(x::AbstractVector, y::Number) where {T} map(a -> l(a, y), x)::AbstractVector end -@inline function (l::PowerNode{T})(x::AbstractVector, y::AbstractVector) where T +@inline function (l::PowerNode{T})(x::AbstractVector, y::AbstractVector) where {T} map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::MinNode{T})(x::AbstractVector, y::AbstractVector) where T +@inline function (l::MinNode{T})(x::AbstractVector, y::AbstractVector) where {T} map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::MaxNode{T})(x::AbstractVector, y::AbstractVector) where T +@inline function (l::MaxNode{T})(x::AbstractVector, y::AbstractVector) where {T} map((a, b) -> l(a, b), x, y)::AbstractVector end -@inline function (l::TraceNode{T})(x::AbstractVector) where T +@inline function (l::TraceNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::DeterminantNode{T})(x::AbstractVector) where T +@inline function (l::DeterminantNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::SymmetricNode{T})(x::AbstractVector) where T +@inline function (l::SymmetricNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::SkewNode{T})(x::AbstractVector) where T +@inline function (l::SkewNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::VolumetricNode{T})(x::AbstractVector) where T +@inline function (l::VolumetricNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::DeviatricNode{T})(x::AbstractVector) where T +@inline function (l::DeviatricNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::InversionNode{T})(x::AbstractVector) where T +@inline function (l::InversionNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::TdotNode{T})(x::AbstractVector) where T +@inline function (l::TdotNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::DottNode{T})(x::AbstractVector) where T +@inline function (l::DottNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end -@inline function (l::DeviatoricNode{T})(x::AbstractVector) where T +@inline function (l::DeviatoricNode{T})(x::AbstractVector) where {T} map(l, x)::AbstractVector end @@ -359,4 +359,51 @@ export compile_to_flux_network export TENSOR_NODES, TENSOR_NODES_ARITY +@setup_workload begin + dim = 3 + t2 = rand(Tensor{2,dim}) + vec3 = rand(Tensor{1,dim}) + const_val = 2.0 + nodes = OrderedDict{Int8,Any}( + 5 => InputSelector(1), + 6 => InputSelector(2), + 7 => const_val + ) + arity_map = OrderedDict{Int8,Int}( + 1 => 2, # + (AdditionNode) + 2 => 2, # * (MultiplicationNode) + 3 => 2, # dcontract + 4 => 1 # tr + ) + callbacks = Dict{Int8,Any}( + 1 => AdditionNode, + 2 => MultiplicationNode, + 3 => DoubleContractionNode, + 4 => TraceNode + ) + + expressions = [ + Int8[2, 5, 5], + Int8[2, 5, 5], + Int8[1, 5, 5], + Int8[2, 6, 7], + Int8[1, 5, 6], + Int8[3, 5, 5], + Int8[4, 5] + ] + inputs = (t2, vec3, const_val) + inputs2 = ([t2 for _ in 1:10],[vec3 for _ in 1:10],[const_val for _ in 1:10]) + @compile_workload begin + for expr in expressions + net = compile_to_flux_network(expr, arity_map, callbacks, nodes, 0) + try + net(inputs) + net(inputs2) + catch + # Ignore runtime dimension mismatch for precompile + end + end + end +end + end \ No newline at end of file From 2ca242fbcfd18eb625bbf93cb51da4fcb3f93503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Rei=C3=9Fmann?= Date: Thu, 30 Jan 2025 22:37:11 +1100 Subject: [PATCH 41/41] remark --- paper/ConstraintSymbolicRegression.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paper/ConstraintSymbolicRegression.md b/paper/ConstraintSymbolicRegression.md index a562296..a82a1eb 100644 --- a/paper/ConstraintSymbolicRegression.md +++ b/paper/ConstraintSymbolicRegression.md @@ -3,7 +3,8 @@ Evolutionary symbolic regression approaches are powerful tools that can approxim # Test Reproduction -The file ConstrainViaSBP.jl contains the test setup. Please follow the steps outlined there. +- The file ConstrainViaSBP.jl contains the test setup. Please follow the steps outlined there. +- Utilize release 0.4 # Source - [1] Reissmann, M., Fang, Y., Ooi, A., & Sandberg, R. (2024). Constraining genetic symbolic regression via semantic backpropagation. arXiv. https://arxiv.org/abs/2409.07369 \ No newline at end of file