From 30083a6d9817dc340ecac2413adbf0ee1f35c1b1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 12:48:36 -0700 Subject: [PATCH 001/174] const SEM=StructuralEquationModels for resolving the scope --- src/StructuralEquationModels.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 131ec206f..05d077662 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -9,6 +9,8 @@ using LinearAlgebra, Optim, import DataFrames: DataFrame export StenoGraphs, @StenoGraph, meld +const SEM = StructuralEquationModels + # type hierarchy include("types.jl") include("objective_gradient_hessian.jl") From a505da0b8193c3fc9df4b3c2dcf878bc7c8d3d5e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 11 Mar 2024 22:49:16 -0700 Subject: [PATCH 002/174] ParamTable: convert from Dict to NamedTuple * both columns and vars * remove _vars suffix from variable categories ("variables" is already a field name) --- src/StructuralEquationModels.jl | 1 - src/frontend/specification/ParameterTable.jl | 173 +++++++++---------- src/frontend/specification/RAMMatrices.jl | 87 ++++------ src/observed/get_colnames.jl | 13 +- test/examples/helper.jl | 4 +- test/unit_tests/data_input_formats.jl | 5 +- 6 files changed, 128 insertions(+), 155 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 05d077662..63ccae9dd 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -6,7 +6,6 @@ using LinearAlgebra, Optim, Distributions, StenoGraphs, LazyArtifacts, DelimitedFiles, DataFrames -import DataFrames: DataFrame export StenoGraphs, @StenoGraph, meld const SEM = StructuralEquationModels diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 1ea4f46f4..006d3acbd 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -14,24 +14,24 @@ end ############################################################################################ # constuct an empty table -function ParameterTable(::Nothing) - - columns = Dict{Symbol, Any}( - :from => Vector{Symbol}(), - :parameter_type => Vector{Symbol}(), - :to => Vector{Symbol}(), - :free => Vector{Bool}(), - :value_fixed => Vector{Float64}(), - :start => Vector{Float64}(), - :estimate => Vector{Float64}(), - :identifier => Vector{Symbol}(), - :start => Vector{Float64}(), +function ParameterTable(; observed_vars::Union{AbstractVector{Symbol}, Nothing}=nothing, + latent_vars::Union{AbstractVector{Symbol}, Nothing}=nothing) + columns = ( + from = Vector{Symbol}(), + parameter_type = Vector{Symbol}(), + to = Vector{Symbol}(), + free = Vector{Bool}(), + value_fixed = Vector{Float64}(), + start = Vector{Float64}(), + estimate = Vector{Float64}(), + se = Vector{Float64}(), + identifier = Vector{Symbol}(), ) - variables = Dict{Symbol, Any}( - :latent_vars => Vector{Symbol}(), - :observed_vars => Vector{Symbol}(), - :sorted_vars => Vector{Symbol}() + variables = ( + latent = !isnothing(latent_vars) ? copy(latent_vars) : Vector{Symbol}(), + observed = !isnothing(observed_vars) ? copy(observed_vars) : Vector{Symbol}(), + sorted = Vector{Symbol}() ) return ParameterTable(columns, variables) @@ -41,18 +41,19 @@ end ### Convert to other types ############################################################################################ -import Base.Dict - -function Dict(partable::ParameterTable) +function Base.convert(::Type{Dict}, partable::ParameterTable) return partable.columns end -function DataFrame( - partable::ParameterTable; - columns = nothing) - if isnothing(columns) columns = keys(partable.columns) end - out = DataFrame([key => partable.columns[key] for key in columns]) - return DataFrame(out) +function DataFrames.DataFrame( + partable::ParameterTable; + columns::Union{AbstractVector{Symbol}, Nothing} = nothing) + if isnothing(columns) + columns = [col for (col, vals) in pairs(partable.columns) + if length(vals) > 0] + end + return DataFrame([col => partable.columns[col] + for col in columns]) end ############################################################################################ @@ -70,24 +71,21 @@ function Base.show(io::IO, partable::ParameterTable) :estimate, :se, :identifier] - existing_columns = [haskey(partable.columns, key) for key in relevant_columns] - - as_matrix = hcat([partable.columns[key] for key in relevant_columns[existing_columns]]...) + shown_columns = filter!(col -> haskey(partable.columns, col) && length(partable.columns[col]) > 0, + relevant_columns) + + as_matrix = mapreduce(col -> partable.columns[col], hcat, shown_columns) pretty_table( - io, + io, as_matrix, header = ( - relevant_columns[existing_columns], - eltype.([partable.columns[key] for key in relevant_columns[existing_columns]]) + shown_columns, + [eltype(partable.columns[col]) for col in shown_columns] ), tf = PrettyTables.tf_compact) - if haskey(partable.variables, :latent_vars) - print(io, "Latent Variables: $(partable.variables[:latent_vars]) \n") - end - if haskey(partable.variables, :observed_vars) - print(io, "Observed Variables: $(partable.variables[:observed_vars]) \n") - end + print(io, "Latent Variables: $(partable.variables.latent) \n") + print(io, "Observed Variables: $(partable.variables.observed) \n") end ############################################################################################ @@ -95,23 +93,16 @@ end ############################################################################################ # Iteration -------------------------------------------------------------------------------- +Base.getindex(partable::ParameterTable, i::Integer) = + (from = partable.columns.from[i], + parameter_type = partable.columns.parameter_type[i], + to = partable.columns.to[i], + free = partable.columns.free[i], + value_fixed = partable.columns.value_fixed[i], + identifier = partable.columns.identifier[i], + ) -Base.getindex(partable::ParameterTable, i::Int) = - (partable.columns[:from][i], - partable.columns[:parameter_type][i], - partable.columns[:to][i], - partable.columns[:free][i], - partable.columns[:value_fixed][i], - partable.columns[:identifier][i]) - -function Base.length(partable::ParameterTable) - len = missing - for key in keys(partable.columns) - len = length(partable.columns[key]) - break - end - return len -end +Base.length(partable::ParameterTable) = length(first(partable.columns)) # Sorting ---------------------------------------------------------------------------------- @@ -121,47 +112,45 @@ end Base.showerror(io::IO, e::CyclicModelError) = print(io, e.msg) -import Base.sort!, Base.sort +function Base.sort!(partable::ParameterTable) -function sort!(partable::ParameterTable) + vars = [partable.variables.latent; + partable.variables.observed] - variables = [partable.variables[:latent_vars]; partable.variables[:observed_vars]] + is_regression = [(partype == :→) && (from != Symbol("1")) + for (partype, from) in zip(partable.columns.parameter_type, + partable.columns.from)] - is_regression = (partable.columns[:parameter_type] .== :→) .& (partable.columns[:from] .!= Symbol("1")) + to = partable.columns.to[is_regression] + from = partable.columns.from[is_regression] - to = partable.columns[:to][is_regression] - from = partable.columns[:from][is_regression] + sorted_vars = Vector{Symbol}() - sorted_variables = Vector{Symbol}() + while !isempty(vars) - sorted = false - while !sorted - acyclic = false - - for (i, variable) in enumerate(variables) - if !(variable ∈ to) - push!(sorted_variables, variable) - deleteat!(variables, i) - delete_edges = from .!= variable + + for (i, var) in enumerate(vars) + if !(var ∈ to) + push!(sorted_vars, var) + deleteat!(vars, i) + delete_edges = from .!= var to = to[delete_edges] from = from[delete_edges] acyclic = true end end - - if !acyclic throw(CyclicModelError("your model is cyclic and therefore can not be ordered")) end - acyclic = false - if length(variables) == 0 sorted = true end + acyclic || throw(CyclicModelError("your model is cyclic and therefore can not be ordered")) end - push!(partable.variables, :sorted_vars => sorted_variables) + copyto!(resize!(partable.variables.sorted, length(sorted_vars)), + sorted_vars) return partable end -function sort(partable::ParameterTable) +function Base.sort(partable::ParameterTable) new_partable = deepcopy(partable) sort!(new_partable) return new_partable @@ -169,15 +158,13 @@ end # add a row -------------------------------------------------------------------------------- -import Base.push! - -function push!(partable::ParameterTable, d::AbstractDict) +function Base.push!(partable::ParameterTable, d::NamedTuple) for key in keys(d) push!(partable.columns[key], d[key]) end end -push!(partable::ParameterTable, d::Nothing) = nothing +Base.push!(partable::ParameterTable, d::Nothing) = nothing ############################################################################################ ### Update Partable from Fitted Model @@ -185,29 +172,33 @@ push!(partable::ParameterTable, d::Nothing) = nothing # update generic --------------------------------------------------------------------------- -function update_partable!(partable::ParameterTable, model_identifier::AbstractDict, vec, column) - new_col = Vector{eltype(vec)}(undef, length(partable)) - for (i, identifier) in enumerate(partable.columns[:identifier]) - if !(identifier == :const) - new_col[i] = vec[model_identifier[identifier]] - elseif identifier == :const - new_col[i] = zero(eltype(vec)) +function update_partable!(partable::ParameterTable, + model_identifier::AbstractDict, + values::AbstractVector, + column::Symbol) + coldata = partable.columns[column] + resize!(coldata, length(partable)) + for (i, id) in enumerate(partable.columns.identifier) + if !(id == :const) + coldata[i] = values[model_identifier[id]] + elseif id == :const + coldata[i] = zero(eltype(values)) end end - push!(partable.columns, column => new_col) return partable end """ - update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, vec, column) - + update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, values, column) + Write `vec` to `column` of `partable`. # Arguments - `vec::Vector`: has to be in the same order as the `model` parameters """ -update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, vec, column) = - update_partable!(partable, identifier(sem_fit), vec, column) +update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, + values::AbstractVector, column::Symbol) = + update_partable!(partable, identifier(sem_fit), values, column) # update estimates ------------------------------------------------------------------------- """ diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 203e389d8..d850febf8 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -103,26 +103,23 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) par_id[:parameters], par_id[:n_par], par_id[:par_positions] end - n_observed = size(partable.variables[:observed_vars], 1) - n_latent = size(partable.variables[:latent_vars], 1) + n_observed = length(partable.variables.observed) + n_latent = length(partable.variables.latent) n_node = n_observed + n_latent # F indices - if length(partable.variables[:sorted_vars]) != 0 - F_ind = findall(x -> x ∈ partable.variables[:observed_vars], partable.variables[:sorted_vars]) - else - F_ind = 1:n_observed - end + F_ind = length(partable.variables.sorted) != 0 ? + findall(∈(Set(partable.variables.observed)), + partable.variables.sorted) : + 1:n_observed # indices of the colnames - if length(partable.variables[:sorted_vars]) != 0 - positions = Dict(zip(partable.variables[:sorted_vars], collect(1:n_observed+n_latent))) - colnames = copy(partable.variables[:sorted_vars]) - else - positions = Dict(zip([partable.variables[:observed_vars]; partable.variables[:latent_vars]], collect(1:n_observed+n_latent))) - colnames = [partable.variables[:observed_vars]; partable.variables[:latent_vars]] - end - + colnames = length(partable.variables.sorted) != 0 ? + copy(partable.variables.sorted) : + [partable.variables.observed; + partable.variables.latent] + positions = Dict(colnames .=> eachindex(colnames)) + # fill Matrices # known_labels = Dict{Symbol, Int64}() @@ -132,43 +129,40 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) for i in 1:length(S_ind) S_ind[i] = Vector{Int64}() end # is there a meanstructure? - if any(partable.columns[:from] .== Symbol("1")) - M_ind = Vector{Vector{Int64}}(undef, n_par) - for i in 1:length(M_ind) M_ind[i] = Vector{Int64}() end - else - M_ind = nothing - end + M_ind = any(==(Symbol("1")), partable.columns.from) ? + [Vector{Int64}() for _ in 1:n_par] : nothing - # handel constants + # handle constants constants = Vector{RAMConstant}() - - for i in 1:length(partable) - from, parameter_type, to, free, value_fixed, identifier = partable[i] + for row in partable - row_ind = positions[to] - if from != Symbol("1") col_ind = positions[from] end - + row_ind = positions[row.to] + col_ind = row.from != Symbol("1") ? positions[row.from] : nothing - if !free - if (parameter_type == :→) & (from == Symbol("1")) - push!(constants, RAMConstant(:M, row_ind, value_fixed)) - elseif (parameter_type == :→) - push!(constants, RAMConstant(:A, CartesianIndex(row_ind, col_ind), value_fixed)) + if !row.free + if (row.parameter_type == :→) && (row.from == Symbol("1")) + push!(constants, RAMConstant(:M, row_ind, row.value_fixed)) + elseif (row.parameter_type == :→) + push!(constants, RAMConstant(:A, CartesianIndex(row_ind, col_ind), row.value_fixed)) + elseif (row.parameter_type == :↔) + push!(constants, RAMConstant(:S, CartesianIndex(row_ind, col_ind), row.value_fixed)) else - push!(constants, RAMConstant(:S, CartesianIndex(row_ind, col_ind), value_fixed)) + error("Unsupported parameter type: $(row.parameter_type)") end else - par_ind = par_positions[identifier] - if (parameter_type == :→) && (from == Symbol("1")) + par_ind = par_positions[row.identifier] + if (row.parameter_type == :→) && (row.from == Symbol("1")) push!(M_ind[par_ind], row_ind) - elseif parameter_type == :→ - push!(A_ind[par_ind], (row_ind + (col_ind-1)*n_node)) - else + elseif row.parameter_type == :→ + push!(A_ind[par_ind], row_ind + (col_ind-1)*n_node) + elseif row.parameter_type == :↔ push!(S_ind[par_ind], row_ind + (col_ind-1)*n_node) if row_ind != col_ind push!(S_ind[par_ind], col_ind + (row_ind-1)*n_node) end + else + error("Unsupported parameter type: $(row.parameter_type)") end end @@ -182,21 +176,14 @@ end ############################################################################################ function ParameterTable(ram_matrices::RAMMatrices) - - partable = ParameterTable(nothing) colnames = ram_matrices.colnames - position_names = Dict{Int64, Symbol}(1:length(colnames) .=> colnames) - # observed and latent variables - names_obs = colnames[ram_matrices.F_ind] - names_lat = colnames[findall(x -> !(x ∈ ram_matrices.F_ind), 1:length(colnames))] + partable = ParameterTable(observed_vars = colnames[ram_matrices.F_ind], + latent_vars = colnames[setdiff(eachindex(colnames), + ram_matrices.F_ind)]) - partable.variables = Dict( - :sorted_vars => Vector{Symbol}(), - :observed_vars => names_obs, - :latent_vars => names_lat - ) + position_names = Dict{Int64, Symbol}(1:length(colnames) .=> colnames) # constants for c in ram_matrices.constants diff --git a/src/observed/get_colnames.jl b/src/observed/get_colnames.jl index 19d6e5a16..599588ce8 100644 --- a/src/observed/get_colnames.jl +++ b/src/observed/get_colnames.jl @@ -1,12 +1,9 @@ -# specification colnames +# specification colnames (only observed) function get_colnames(specification::ParameterTable) - if !haskey(specification.variables, :sorted_vars) || - (length(specification.variables[:sorted_vars]) == 0) - colnames = specification.variables[:observed_vars] - else - is_obs = [var ∈ specification.variables[:observed_vars] for var in specification.variables[:sorted_vars]] - colnames = specification.variables[:sorted_vars][is_obs] - end + colnames = isempty(specification.variables.sorted) ? + specification.variables.observed : + filter(in(Set(specification.variables.observed)), + specification.variables.sorted) return colnames end diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 166861b08..9cdf18bf7 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -110,7 +110,7 @@ function compare_estimates(partable::ParameterTable, partable_lav; if type == :↔ type = "~~" elseif type == :→ - if (from ∈ partable.variables[:latent_vars]) & (to ∈ partable.variables[:observed_vars]) + if (from ∈ partable.variables.latent) && (to ∈ partable.variables.observed) type = "=~" else type = "~" @@ -207,7 +207,7 @@ function compare_estimates(ens_partable::EnsembleParameterTable, partable_lav; if type == :↔ type = "~~" elseif type == :→ - if (from ∈ partable.variables[:latent_vars]) & (to ∈ partable.variables[:observed_vars]) + if (from ∈ partable.variables.latent) && (to ∈ partable.variables.observed) type = "=~" else type = "~" diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 77aed035e..992140f57 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -3,9 +3,8 @@ import StructuralEquationModels: obs_cov, obs_mean, get_data ### model specification -------------------------------------------------------------------- -spec = ParameterTable(nothing) -spec.variables[:observed_vars] = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8] -spec.variables[:latent_vars] = [:ind60, :dem60, :dem65] +spec = ParameterTable(observed_vars = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8], + latent_vars = [:ind60, :dem60, :dem65]) ### data ----------------------------------------------------------------------------------- From 0d0d389368ff819236ed8379da9057b9a59718c3 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 13:10:56 -0800 Subject: [PATCH 003/174] obj_grad_hess: simplify mapreduce * destructure lambda function args * don't use zip(): mapreduce support multi-arg functions --- src/objective_gradient_hessian.jl | 54 +++++++++++++------------------ 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index 8d33ad804..b1119afd2 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -94,10 +94,9 @@ end function objective!(loss::SemLoss, par, model) return mapreduce( - fun_weight -> fun_weight[2]*objective!(fun_weight[1], par, model), - +, - zip(loss.functions, loss.weights) - ) + (fun, weight) -> weight*objective!(fun, par, model), + +, + loss.functions, loss.weights) end function gradient!(gradient, loss::SemLoss, par, model) @@ -115,18 +114,16 @@ end function objective_gradient!(gradient, loss::SemLoss, par, model) return mapreduce( - fun_weight -> objective_gradient_wrap_(gradient, fun_weight[1], par, model, fun_weight[2]), - +, - zip(loss.functions, loss.weights) - ) + (fun, weight) -> objective_gradient_wrap_(gradient, fun, par, model, weight), + +, + loss.functions, loss.weights) end function objective_hessian!(hessian, loss::SemLoss, par, model) return mapreduce( - fun_weight -> objective_hessian_wrap_(hessian, fun_weight[1], par, model, fun_weight[2]), - +, - zip(loss.functions, loss.weights) - ) + (fun, weight) -> objective_hessian_wrap_(hessian, fun, par, model, weight), + +, + loss.functions, loss.weights) end function gradient_hessian!(gradient, hessian, loss::SemLoss, par, model) @@ -139,10 +136,9 @@ end function objective_gradient_hessian!(gradient, hessian, loss::SemLoss, par, model) return mapreduce( - fun_weight -> objective_gradient_hessian_wrap_(gradient, hessian, fun_weight[1], par, model, fun_weight[2]), - +, - zip(loss.functions, loss.weights) - ) + (fun, weight) -> objective_gradient_hessian_wrap_(gradient, hessian, fun, par, model, weight), + +, + loss.functions, loss.weights) end # wrapper to update gradient/hessian and return objective value @@ -171,10 +167,9 @@ end function objective!(ensemble::SemEnsemble, par) return mapreduce( - model_weight -> model_weight[2]*objective!(model_weight[1], par), - +, - zip(ensemble.sems, ensemble.weights) - ) + (model, weight) -> weight*objective!(model, par), + +, + ensemble.sems, ensemble.weights) end function gradient!(gradient, ensemble::SemEnsemble, par) @@ -198,19 +193,17 @@ end function objective_gradient!(gradient, ensemble::SemEnsemble, par) fill!(gradient, zero(eltype(gradient))) return mapreduce( - model_weight -> objective_gradient_wrap_(gradient, model_weight[1], par, model_weight[2]), - +, - zip(ensemble.sems, ensemble.weights) - ) + (model, weight) -> objective_gradient_wrap_(gradient, model, par, weight), + +, + ensemble.sems, ensemble.weights) end function objective_hessian!(hessian, ensemble::SemEnsemble, par) fill!(hessian, zero(eltype(hessian))) return mapreduce( - model_weight -> objective_hessian_wrap_(hessian, model_weight[1], par, model_weight[2]), + (model, weight) -> objective_hessian_wrap_(hessian, model, par, weight), +, - zip(ensemble.sems, ensemble.weights) - ) + ensemble.sems, ensemble.weights) end function gradient_hessian!(gradient, hessian, ensemble::SemEnsemble, par) @@ -233,10 +226,9 @@ function objective_gradient_hessian!(gradient, hessian, ensemble::SemEnsemble, p fill!(gradient, zero(eltype(gradient))) fill!(hessian, zero(eltype(hessian))) return mapreduce( - model_weight -> objective_gradient_hessian_wrap_(gradient, hessian, model_weight[1], par, model, model_weight[2]), - +, - zip(ensemble.sems, ensemble.weights) - ) + (model, weight) -> objective_gradient_hessian_wrap_(gradient, hessian, model, par, model, weight), + +, + ensemble.sems, ensemble.weights) end # wrapper to update gradient/hessian and return objective value From 080a8ad14c39de5b09f20bc177b6045ca67509ed Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 18:06:11 -0700 Subject: [PATCH 004/174] cov_and_mean(): use StatsBase.mean_and_cov() --- src/StructuralEquationModels.jl | 2 +- src/additional_functions/helper.jl | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 63ccae9dd..ad0e2ccf6 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -1,7 +1,7 @@ module StructuralEquationModels using LinearAlgebra, Optim, - NLSolversBase, Statistics, SparseArrays, Symbolics, + NLSolversBase, Statistics, StatsBase, SparseArrays, Symbolics, NLopt, FiniteDiff, PrettyTables, Distributions, StenoGraphs, LazyArtifacts, DelimitedFiles, DataFrames diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 91095e071..6505de400 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -114,11 +114,9 @@ function sparse_outer_mul!(C, A, B::Vector, ind) #computes A*S*B -> C, where ind end function cov_and_mean(rows; corrected = false) - data = transpose(reduce(hcat, rows)) - size(rows, 1) > 1 ? - obs_cov = Statistics.cov(data; corrected = corrected) : - obs_cov = reshape([0.0],1,1) - obs_mean = vec(Statistics.mean(data, dims = 1)) + obs_mean, obs_cov = StatsBase.mean_and_cov( + reduce(hcat, rows), 1, + corrected = corrected) return obs_cov, obs_mean end From 5e9db0a9462021947707549eb4625e1937c49c3d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 13:14:09 -0800 Subject: [PATCH 005/174] fill_A_S_M(): use `@inbounds` --- src/additional_functions/parameters.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl index 13aa1cd41..6f24fbb5c 100644 --- a/src/additional_functions/parameters.jl +++ b/src/additional_functions/parameters.jl @@ -1,7 +1,6 @@ function fill_A_S_M(A, S, M, A_indices, S_indices, M_indices, parameters) - for (iA, iS, par) in zip(A_indices, S_indices, parameters) - + @inbounds for (iA, iS, par) in zip(A_indices, S_indices, parameters) for index_A in iA A[index_A] = par end @@ -14,8 +13,7 @@ function fill_A_S_M(A, S, M, A_indices, S_indices, M_indices, parameters) if !isnothing(M) - for (iM, par) in zip(M_indices, parameters) - + @inbounds for (iM, par) in zip(M_indices, parameters) for index_M in iM M[index_M] = par end From 74735f30b0cc12b6a32d226ee0c08f6604e55e95 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 12:23:49 -0700 Subject: [PATCH 006/174] fill_A_S_M!(): add ! to match julia naming convention --- src/additional_functions/parameters.jl | 18 +++++++++++++----- src/imply/RAM/generic.jl | 6 +++--- src/imply/RAM/symbolic.jl | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl index 6f24fbb5c..ede8e865a 100644 --- a/src/additional_functions/parameters.jl +++ b/src/additional_functions/parameters.jl @@ -1,4 +1,12 @@ -function fill_A_S_M(A, S, M, A_indices, S_indices, M_indices, parameters) +# fill A, S, and M matrices with the parameter values according to the parameters map +function fill_A_S_M!( + A::AbstractMatrix, S::AbstractMatrix, + M::Union{AbstractVector, Nothing}, + A_indices::AbstractArrayParamsMap, + S_indices::AbstractArrayParamsMap, + M_indices::Union{AbstractArrayParamsMap, Nothing}, + parameters::AbstractVector +) @inbounds for (iA, iS, par) in zip(A_indices, S_indices, parameters) for index_A in iA @@ -107,16 +115,16 @@ function get_matrix_derivative(M_indices, parameters, n_long) end -function fill_matrix(M, M_indices, parameters) +# fill M with parameters +function fill_matrix!(M::AbstractMatrix, M_indices::AbstractArrayParamsMap, + parameters::AbstractVector) for (iM, par) in zip(M_indices, parameters) - for index_M in iM M[index_M] = par end - end - + return M end function get_partition(A_indices, S_indices) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 95803b858..435fff0eb 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -207,7 +207,7 @@ gradient!(imply::RAM, par, model::AbstractSemSingle) = # objective and gradient function objective!(imply::RAM, parameters, model, has_meanstructure::Val{T}) where T - fill_A_S_M( + fill_A_S_M!( imply.A, imply.S, imply.M, @@ -235,7 +235,7 @@ end function gradient!(imply::RAM, parameters, model::AbstractSemSingle, has_meanstructure::Val{T}) where T - fill_A_S_M( + fill_A_S_M!( imply.A, imply.S, imply.M, @@ -335,7 +335,7 @@ function check_acyclic(A_pre, n_par, A_indices) A_rand = copy(A_pre) randpar = rand(n_par) - fill_matrix( + fill_matrix!( A_rand, A_indices, randpar) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 23362646b..756bf2a91 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -110,7 +110,7 @@ function RAMSymbolic(; F = zeros(ram_matrices.size_F); F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 set_RAMConstants!(A, S, M, ram_matrices.constants) - fill_A_S_M(A, S, M, ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.M_ind, par) + fill_A_S_M!(A, S, M, ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.M_ind, par) A, S, F = sparse(A), sparse(S), sparse(F) From 60c61f3025fa939f4e00e805037cfaaf066263e8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 13:38:48 -0800 Subject: [PATCH 007/174] ==: use && avoids full evalulation when the end result is known --- src/frontend/specification/RAMMatrices.jl | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index d850febf8..ac4ffda93 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -41,9 +41,8 @@ end import Base.== function ==(c1::RAMConstant, c2::RAMConstant) - res = ( (c1.matrix == c2.matrix) & (c1.index == c2.index) & - (c1.value == c2.value) - ) + res = ( (c1.matrix == c2.matrix) && (c1.index == c2.index) && + (c1.value == c2.value) ) return res end @@ -371,12 +370,11 @@ function push_partable_rows!(partable, position_names, par, i, A_ind, S_ind, M_i end function ==(mat1::RAMMatrices, mat2::RAMMatrices) - res = ( (mat1.A_ind == mat2.A_ind) & (mat1.S_ind == mat2.S_ind) & - (mat1.F_ind == mat2.F_ind) & (mat1.M_ind == mat2.M_ind) & - (mat1.parameters == mat2.parameters) & - (mat1.colnames == mat2.colnames) & (mat1.size_F == mat2.size_F) & - (mat1.constants == mat2.constants) - ) + res = ( (mat1.A_ind == mat2.A_ind) && (mat1.S_ind == mat2.S_ind) && + (mat1.F_ind == mat2.F_ind) && (mat1.M_ind == mat2.M_ind) && + (mat1.parameters == mat2.parameters) && + (mat1.colnames == mat2.colnames) && (mat1.size_F == mat2.size_F) && + (mat1.constants == mat2.constants) ) return res end From ae89781ba30e2d26e6c5950cef8279f268c2a252 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 12:06:44 -0700 Subject: [PATCH 008/174] RAMMatrices: cleanup params index * simplify parameters() function to return just a vector of params * RAMMatrices ctor: warn on duplicate params --- src/frontend/specification/RAMMatrices.jl | 69 +++++++++++------------ 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index ac4ffda93..72be3fca9 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -93,13 +93,20 @@ end ### get RAMMatrices from parameter table ############################################################################################ -function RAMMatrices(partable::ParameterTable; par_id = nothing) +function RAMMatrices(partable::ParameterTable; + params::Union{AbstractVector{Symbol}, Nothing} = nothing) - if isnothing(par_id) - parameters, n_par, par_positions = get_par_npar_identifier(partable) - else - parameters, n_par, par_positions = - par_id[:parameters], par_id[:n_par], par_id[:par_positions] + if isnothing(params) + params = parameters(partable) + end + params_index = Dict(param => i for (i, param) in enumerate(params)) + if length(params) != length(params_index) + params_seen = Set{Symbol}() + params_nonunique = Vector{Symbol}() + for par in params + push!(par in params_seen ? params_nonunique : params_seen, par) + end + throw(ArgumentError("Duplicate names in the parameters vector: $(join(params_nonunique, ", "))")) end n_observed = length(partable.variables.observed) @@ -122,14 +129,12 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) # fill Matrices # known_labels = Dict{Symbol, Int64}() - A_ind = Vector{Vector{Int64}}(undef, n_par) - for i in 1:length(A_ind) A_ind[i] = Vector{Int64}() end - S_ind = Vector{Vector{Int64}}(undef, n_par); S_ind .= [Vector{Int64}()] - for i in 1:length(S_ind) S_ind[i] = Vector{Int64}() end + A_ind = [Vector{Int64}() for _ in 1:length(params)] + S_ind = [Vector{Int64}() for _ in 1:length(params)] # is there a meanstructure? M_ind = any(==(Symbol("1")), partable.columns.from) ? - [Vector{Int64}() for _ in 1:n_par] : nothing + [Vector{Int64}() for _ in 1:length(params)] : nothing # handle constants constants = Vector{RAMConstant}() @@ -150,7 +155,7 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) error("Unsupported parameter type: $(row.parameter_type)") end else - par_ind = par_positions[row.identifier] + par_ind = params_index[row.identifier] if (row.parameter_type == :→) && (row.from == Symbol("1")) push!(M_ind[par_ind], row_ind) elseif row.parameter_type == :→ @@ -167,7 +172,9 @@ function RAMMatrices(partable::ParameterTable; par_id = nothing) end - return RAMMatrices(A_ind, S_ind, F_ind, M_ind, parameters, colnames, constants, (n_observed, n_node)) + return RAMMatrices(A_ind, S_ind, F_ind, M_ind, + params, colnames, constants, + (n_observed, n_node)) end ############################################################################################ @@ -212,11 +219,10 @@ function RAMMatrices(partable::EnsembleParameterTable) ram_matrices = Dict{Symbol, RAMMatrices}() - parameters, n_par, par_positions = get_par_npar_identifier(partable) - par_id = Dict(:parameters => parameters, :n_par => n_par, :par_positions => par_positions) + params = parameters(partable) for key in keys(partable.tables) - ram_mat = RAMMatrices(partable.tables[key]; par_id = par_id) + ram_mat = RAMMatrices(partable.tables[key]; params = params) push!(ram_matrices, key => ram_mat) end @@ -236,28 +242,21 @@ end ### Additional Functions ############################################################################################ -function get_par_npar_identifier(partable::ParameterTable) - parameters = unique(partable.columns[:identifier]) - filter!(x -> x != :const, parameters) - n_par = length(parameters) - par_positions = Dict(parameters .=> 1:n_par) - return parameters, n_par, par_positions -end - -function get_par_npar_identifier(partable::EnsembleParameterTable) - - parameters = Vector{Symbol}() - for key in keys(partable.tables) - append!(parameters, partable.tables[key].columns[:identifier]) +# get the vector of all parameters in the table +# the position of the parameter is based on its first appearance in the table (and the ensemble) +function parameters(partable::Union{EnsembleParameterTable, ParameterTable}) + if partable isa ParameterTable + parameters = partable.columns.identifier + else + parameters = Vector{Symbol}() + for tbl in values(partable.tables) + append!(parameters, tbl.columns.identifier) + end end parameters = unique(parameters) - filter!(x -> x != :const, parameters) - - n_par = length(parameters) - - par_positions = Dict(parameters .=> 1:n_par) + filter!(!=(:const), parameters) # exclude constants - return parameters, n_par, par_positions + return parameters end function get_partable_row(c::RAMConstant, position_names) From 4609c00472c07a7b1550a3244c5b961a2f2eeaa4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 14:20:59 -0800 Subject: [PATCH 009/174] RAMMatrices: tiny rename for clarity --- src/frontend/specification/RAMMatrices.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 72be3fca9..2b945345e 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -124,7 +124,7 @@ function RAMMatrices(partable::ParameterTable; copy(partable.variables.sorted) : [partable.variables.observed; partable.variables.latent] - positions = Dict(colnames .=> eachindex(colnames)) + cols_index = Dict(col => i for (i, col) in enumerate(colnames)) # fill Matrices # known_labels = Dict{Symbol, Int64}() @@ -141,8 +141,8 @@ function RAMMatrices(partable::ParameterTable; for row in partable - row_ind = positions[row.to] - col_ind = row.from != Symbol("1") ? positions[row.from] : nothing + row_ind = cols_index[row.to] + col_ind = row.from != Symbol("1") ? cols_index[row.from] : nothing if !row.free if (row.parameter_type == :→) && (row.from == Symbol("1")) From 2739f0b4958dda8faddde3f5a9120390b47bc14b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:06:48 -0800 Subject: [PATCH 010/174] RAMMatrices: cleanup indexing params in arrays * introduce ArrayParamsMap type for clarity * annotate types of the corresp. RAMMatrices fields * rename get_parameter_indices() into array_parameters_map[_linear()] (two distinct functions without kwarg for type stability) and remove cartesian2linear() (not needed) * use the single pass over the array for performance --- src/additional_functions/parameters.jl | 29 +++++++++++++---------- src/frontend/specification/RAMMatrices.jl | 21 +++++++++------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl index ede8e865a..9a21c95a6 100644 --- a/src/additional_functions/parameters.jl +++ b/src/additional_functions/parameters.jl @@ -32,18 +32,26 @@ function fill_A_S_M!( end -function get_parameter_indices(parameters, M; linear = true, kwargs...) - - M_indices = [findall(x -> (x == par), M) for par in parameters] - - if linear - M_indices = cartesian2linear.(M_indices, [M]) +# build the map from the index of the parameter to the indices of this parameter +# occurences in the given array +function array_parameters_map(parameters::AbstractVector, M::AbstractArray) + params_index = Dict(param => i for (i, param) in enumerate(parameters)) + T = Base.eltype(eachindex(M)) + res = [Vector{T}() for _ in eachindex(parameters)] + for (i, val) in pairs(M) + par_ind = get(params_index, val, nothing) + if !isnothing(par_ind) + push!(res[par_ind], i) + end end - - return M_indices - + return res end +# build the map of parameter index to the linear indices of its occurences in M +# returns ArrayParamsMap object +array_parameters_map_linear(parameters::AbstractVector, M::AbstractArray) = + array_parameters_map(parameters, vec(M)) + function eachindex_lower(M; linear_indices = false, kwargs...) indices = CartesianIndices(M) @@ -67,9 +75,6 @@ function linear2cartesian(ind_lin, dims) return ind_cart end -cartesian2linear(ind_cart, A::AbstractArray) = cartesian2linear(ind_cart, size(A)) -linear2cartesian(ind_linear, A::AbstractArray) = linear2cartesian(ind_linear, size(A)) - function set_constants!(M, M_pre) for index in eachindex(M) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 2b945345e..af3c719dd 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -2,11 +2,15 @@ ### Type ############################################################################################ +# map from parameter index to linear indices of matrix/vector positions where it occurs +AbstractArrayParamsMap = AbstractVector{<:AbstractVector{<:Integer}} +ArrayParamsMap = Vector{Vector{Int}} + struct RAMMatrices - A_ind - S_ind - F_ind - M_ind + A_ind::ArrayParamsMap + S_ind::ArrayParamsMap + F_ind::Vector{Int} + M_ind::Union{ArrayParamsMap, Nothing} parameters colnames constants @@ -18,12 +22,13 @@ end ############################################################################################ function RAMMatrices(;A, S, F, M = nothing, parameters, colnames) - A_indices = get_parameter_indices(parameters, A) - S_indices = get_parameter_indices(parameters, S) - isnothing(M) ? M_indices = nothing : M_indices = get_parameter_indices(parameters, M) + A_indices = array_parameters_map_linear(parameters, A) + S_indices = array_parameters_map_linear(parameters, S) + M_indices = !isnothing(M) ? array_parameters_map_linear(parameters, M) : nothing F_indices = findall([any(isone.(col)) for col in eachcol(F)]) constants = get_RAMConstants(A, S, M) - return RAMMatrices(A_indices, S_indices, F_indices, M_indices, parameters, colnames, constants, size(F)) + return RAMMatrices(A_indices, S_indices, F_indices, M_indices, + parameters, colnames, constants, size(F)) end RAMMatrices(a::RAMMatrices) = a From 613dba71b9b4923df59ee6f50a577ad2d341ac19 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:16:21 -0800 Subject: [PATCH 011/174] RAMConstant: move before RAMMatrices RAMMatrices depend on RAMConstant type --- src/frontend/specification/RAMMatrices.jl | 70 +++++++++++------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index af3c719dd..02235d224 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -1,38 +1,3 @@ -############################################################################################ -### Type -############################################################################################ - -# map from parameter index to linear indices of matrix/vector positions where it occurs -AbstractArrayParamsMap = AbstractVector{<:AbstractVector{<:Integer}} -ArrayParamsMap = Vector{Vector{Int}} - -struct RAMMatrices - A_ind::ArrayParamsMap - S_ind::ArrayParamsMap - F_ind::Vector{Int} - M_ind::Union{ArrayParamsMap, Nothing} - parameters - colnames - constants - size_F -end - -############################################################################################ -### Constructor -############################################################################################ - -function RAMMatrices(;A, S, F, M = nothing, parameters, colnames) - A_indices = array_parameters_map_linear(parameters, A) - S_indices = array_parameters_map_linear(parameters, S) - M_indices = !isnothing(M) ? array_parameters_map_linear(parameters, M) : nothing - F_indices = findall([any(isone.(col)) for col in eachcol(F)]) - constants = get_RAMConstants(A, S, M) - return RAMMatrices(A_indices, S_indices, F_indices, M_indices, - parameters, colnames, constants, size(F)) -end - -RAMMatrices(a::RAMMatrices) = a - ############################################################################################ ### Constants ############################################################################################ @@ -94,6 +59,41 @@ function set_RAMConstants!(A, S, M, rc_vec::Vector{RAMConstant}) for rc in rc_vec set_RAMConstant!(A, S, M, rc) end end +############################################################################################ +### Type +############################################################################################ + +# map from parameter index to linear indices of matrix/vector positions where it occurs +AbstractArrayParamsMap = AbstractVector{<:AbstractVector{<:Integer}} +ArrayParamsMap = Vector{Vector{Int}} + +struct RAMMatrices + A_ind::ArrayParamsMap + S_ind::ArrayParamsMap + F_ind::Vector{Int} + M_ind::Union{ArrayParamsMap, Nothing} + parameters + colnames + constants + size_F +end + +############################################################################################ +### Constructor +############################################################################################ + +function RAMMatrices(;A, S, F, M = nothing, parameters, colnames) + A_indices = array_parameters_map_linear(parameters, A) + S_indices = array_parameters_map_linear(parameters, S) + M_indices = !isnothing(M) ? array_parameters_map_linear(parameters, M) : nothing + F_indices = findall([any(isone.(col)) for col in eachcol(F)]) + constants = get_RAMConstants(A, S, M) + return RAMMatrices(A_indices, S_indices, F_indices, M_indices, + parameters, colnames, constants, size(F)) +end + +RAMMatrices(a::RAMMatrices) = a + ############################################################################################ ### get RAMMatrices from parameter table ############################################################################################ From 436d31cdfd4435ec108e6a67e4106b6732d11f6b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:23:26 -0800 Subject: [PATCH 012/174] RAMConstant: simplify * declare RAMConstant field types * refactor constants collection to avoid code duplication --- src/frontend/specification/RAMMatrices.jl | 41 +++++++---------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 02235d224..115a0b077 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -3,8 +3,8 @@ ############################################################################################ struct RAMConstant - matrix - index + matrix::Symbol + index::CartesianIndex value end @@ -16,32 +16,14 @@ function ==(c1::RAMConstant, c2::RAMConstant) return res end -function get_RAMConstants(A, S, M) - - constants = Vector{RAMConstant}() - - for index in CartesianIndices(A) - if (A[index] isa Number) && !iszero(A[index]) - push!(constants, RAMConstant(:A, index, A[index])) - end - end - - for index in CartesianIndices(S) - if (S[index] isa Number) && !iszero(S[index]) - push!(constants, RAMConstant(:S, index, S[index])) - end - end - - if !isnothing(M) - for index in CartesianIndices(M) - if (M[index] isa Number) && !iszero(M[index]) - push!(constants, RAMConstant(:M, index, M[index])) - end +function append_RAMConstants!(constants::AbstractVector{RAMConstant}, + mtx_name::Symbol, mtx::AbstractArray) + for (index, val) in pairs(mtx) + if isa(val, Number) && !iszero(val) + push!(constants, RAMConstant(mtx_name, index, val)) end end - return constants - end function set_RAMConstant!(A, S, M, rc::RAMConstant) @@ -49,7 +31,7 @@ function set_RAMConstant!(A, S, M, rc::RAMConstant) A[rc.index] = rc.value elseif rc.matrix == :S S[rc.index] = rc.value - S[rc.index[2], rc.index[1]] = rc.value + S[rc.index[2], rc.index[1]] = rc.value # symmetric elseif rc.matrix == :M M[rc.index] = rc.value end @@ -74,7 +56,7 @@ struct RAMMatrices M_ind::Union{ArrayParamsMap, Nothing} parameters colnames - constants + constants::Vector{RAMConstant} size_F end @@ -87,7 +69,10 @@ function RAMMatrices(;A, S, F, M = nothing, parameters, colnames) S_indices = array_parameters_map_linear(parameters, S) M_indices = !isnothing(M) ? array_parameters_map_linear(parameters, M) : nothing F_indices = findall([any(isone.(col)) for col in eachcol(F)]) - constants = get_RAMConstants(A, S, M) + constants = Vector{RAMConstant}() + append_RAMConstants!(constants, :A, A) + append_RAMConstants!(constants, :S, S) + isnothing(M) || append_RAMConstants!(constants, :M, M) return RAMMatrices(A_indices, S_indices, F_indices, M_indices, parameters, colnames, constants, size(F)) end From 6ce98c33f8a0c07ec538768327271b0cb06ab625 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:24:39 -0800 Subject: [PATCH 013/174] RAMMatrices: optimize F_indices init --- src/frontend/specification/RAMMatrices.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 115a0b077..df6a704c2 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -68,7 +68,7 @@ function RAMMatrices(;A, S, F, M = nothing, parameters, colnames) A_indices = array_parameters_map_linear(parameters, A) S_indices = array_parameters_map_linear(parameters, S) M_indices = !isnothing(M) ? array_parameters_map_linear(parameters, M) : nothing - F_indices = findall([any(isone.(col)) for col in eachcol(F)]) + F_indices = [i for (i, col) in zip(axes(F, 2), eachcol(F)) if any(isone, col)] constants = Vector{RAMConstant}() append_RAMConstants!(constants, :A, A) append_RAMConstants!(constants, :S, S) From 80b1927e90905a62c56bc118b69c42e587f89aa4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:25:26 -0800 Subject: [PATCH 014/174] RAMMatrices ctor: dim checks --- src/frontend/specification/RAMMatrices.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index df6a704c2..507f977cf 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -64,7 +64,16 @@ end ### Constructor ############################################################################################ -function RAMMatrices(;A, S, F, M = nothing, parameters, colnames) +function RAMMatrices(; A::AbstractMatrix, S::AbstractMatrix, + F::AbstractMatrix, M::Union{AbstractVector, Nothing} = nothing, + parameters::AbstractVector{Symbol}, + colnames::AbstractVector{Symbol}) + ncols = length(colnames) + size(A, 1) == size(A, 2) || throw(DimensionMismatch("A must be a square matrix")) + size(S, 1) == size(S, 2) || throw(DimensionMismatch("S must be a square matrix")) + size(A, 2) == ncols || throw(DimensionMismatch("A should have as many rows and columns as colnames length ($(length(colnames))), $(size(A)) found")) + size(S, 2) == ncols || throw(DimensionMismatch("S should have as many rows and columns as colnames length ($(length(colnames))), $(size(S)) found")) + size(F, 2) == ncols || throw(DimensionMismatch("F should have as many columns as colnames length ($(length(colnames))), $(size(F, 2)) found")) A_indices = array_parameters_map_linear(parameters, A) S_indices = array_parameters_map_linear(parameters, S) M_indices = !isnothing(M) ? array_parameters_map_linear(parameters, M) : nothing From 67e01ce94e15d509b952aa7aa76420c6a6675ca2 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:26:01 -0800 Subject: [PATCH 015/174] RAMMatrices: declare types for all fields --- src/frontend/specification/RAMMatrices.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 507f977cf..bab6b8811 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -54,10 +54,10 @@ struct RAMMatrices S_ind::ArrayParamsMap F_ind::Vector{Int} M_ind::Union{ArrayParamsMap, Nothing} - parameters - colnames + parameters::Vector{Symbol} + colnames::Vector{Symbol} constants::Vector{RAMConstant} - size_F + size_F::Tuple{Int, Int} end ############################################################################################ From b3e12a014bffa06f0edf578e4e5ed12b58d96a7b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 17:19:57 -0700 Subject: [PATCH 016/174] include RAMMatrices before EnsParTable --- src/StructuralEquationModels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index ad0e2ccf6..c7b2750f5 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -17,8 +17,8 @@ include("objective_gradient_hessian.jl") include("frontend/fit/SemFit.jl") # specification of models include("frontend/specification/ParameterTable.jl") -include("frontend/specification/EnsembleParameterTable.jl") include("frontend/specification/RAMMatrices.jl") +include("frontend/specification/EnsembleParameterTable.jl") include("frontend/specification/StenoGraphs.jl") include("frontend/fit/summary.jl") # pretty printing From b3b8a8af8199443557a313d515d0e60cf4ac28eb Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:35:14 -0700 Subject: [PATCH 017/174] fix EnsParTable to Dict{RAMMatrices} convert * this method is not RAMMatrices ctor, it is Dict{K, RAMMatrices} convert * use comprehension to construct dict --- .../specification/EnsembleParameterTable.jl | 12 ++++++++++++ src/frontend/specification/RAMMatrices.jl | 18 ------------------ test/examples/multigroup/build_models.jl | 2 +- test/examples/multigroup/multigroup.jl | 4 ++-- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 8504be567..7d369e225 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -26,6 +26,18 @@ function Dict(partable::EnsembleParameterTable) return partable.tables end +function Base.convert(::Type{Dict{K, RAMMatrices}}, + partables::EnsembleParameterTable; + params::Union{AbstractVector{Symbol}, Nothing} = nothing) where K + + isnothing(params) || (params = SEM.params(partables)) + + return Dict{K, RAMMatrices}( + K(key) => RAMMatrices(partable; params = params) + for (key, partable) in pairs(partables.tables) + ) +end + #= function DataFrame( partable::ParameterTable; columns = nothing) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index bab6b8811..eeda1926f 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -210,24 +210,6 @@ function ParameterTable(ram_matrices::RAMMatrices) end -############################################################################################ -### get RAMMatrices from EnsembleParameterTable -############################################################################################ - -function RAMMatrices(partable::EnsembleParameterTable) - - ram_matrices = Dict{Symbol, RAMMatrices}() - - params = parameters(partable) - - for key in keys(partable.tables) - ram_mat = RAMMatrices(partable.tables[key]; params = params) - push!(ram_matrices, key => ram_mat) - end - - return ram_matrices -end - ############################################################################################ ### Pretty Printing ############################################################################################ diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 4738ab4a6..7824efdb1 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -53,7 +53,7 @@ end partable_s = sort(partable) -specification_s = RAMMatrices(partable_s) +specification_s = convert(Dict{Symbol, RAMMatrices}, partable_s) specification_g1_s = specification_s[:Pasteur] specification_g2_s = specification_s[:Grant_White] diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 5ef6bd8c6..fc40e36d4 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -97,7 +97,7 @@ partable = EnsembleParameterTable(; latent_vars = latent_vars, groups = [:Pasteur, :Grant_White]) -specification = RAMMatrices(partable) +specification = convert(Dict{Symbol, RAMMatrices}, partable) specification_g1 = specification[:Pasteur] specification_g2 = specification[:Grant_White] @@ -125,7 +125,7 @@ partable_miss = EnsembleParameterTable(; latent_vars = latent_vars, groups = [:Pasteur, :Grant_White]) -specification_miss = RAMMatrices(partable_miss) +specification_miss = convert(Dict{Symbol, RAMMatrices}, partable_miss) specification_miss_g1 = specification_miss[:Pasteur] specification_miss_g2 = specification_miss[:Grant_White] From 05040e938310b9062f8deece0bd1e0bc4b69b422 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 18:40:35 -0700 Subject: [PATCH 018/174] replace no-op ctors with convert(T, obj) convert() is a proper method to call to avoid unnecessary construction, ctor semantics requires that a new object is constructed --- src/frontend/specification/EnsembleParameterTable.jl | 6 ++---- src/frontend/specification/RAMMatrices.jl | 5 +++-- src/imply/RAM/generic.jl | 2 +- src/imply/RAM/symbolic.jl | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 7d369e225..f2a38c56d 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -20,10 +20,8 @@ end ### Convert to other types ############################################################################################ -import Base.Dict - -function Dict(partable::EnsembleParameterTable) - return partable.tables +function Base.convert(::Type{Dict}, partable::EnsembleParameterTable) + return convert(Dict, partable.tables) end function Base.convert(::Type{Dict{K, RAMMatrices}}, diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index eeda1926f..af55b2986 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -86,8 +86,6 @@ function RAMMatrices(; A::AbstractMatrix, S::AbstractMatrix, parameters, colnames, constants, size(F)) end -RAMMatrices(a::RAMMatrices) = a - ############################################################################################ ### get RAMMatrices from parameter table ############################################################################################ @@ -176,6 +174,8 @@ function RAMMatrices(partable::ParameterTable; (n_observed, n_node)) end +Base.convert(::Type{RAMMatrices}, partable::ParameterTable) = RAMMatrices(partable) + ############################################################################################ ### get parameter table from RAMMatrices ############################################################################################ @@ -209,6 +209,7 @@ function ParameterTable(ram_matrices::RAMMatrices) return partable end +Base.convert(::Type{<:ParameterTable}, ram_matrices::RAMMatrices) = ParameterTable(ram_matrices) ############################################################################################ ### Pretty Printing diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 435fff0eb..3ac3c86b6 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -106,7 +106,7 @@ function RAM(; meanstructure = false, kwargs...) - ram_matrices = RAMMatrices(specification) + ram_matrices = convert(RAMMatrices, specification) identifier = StructuralEquationModels.identifier(ram_matrices) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 756bf2a91..51f8fe7e6 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -96,7 +96,7 @@ function RAMSymbolic(; approximate_hessian = false, kwargs...) - ram_matrices = RAMMatrices(specification) + ram_matrices = convert(RAMMatrices, specification) identifier = StructuralEquationModels.identifier(ram_matrices) n_par = length(ram_matrices.parameters) From 6187f242144da1ef4e547f7a777a3e2527d10d61 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:28:47 -0800 Subject: [PATCH 019/174] ParTable ctor: simplify rows code * use named tuples * reduce code duplication --- src/frontend/specification/RAMMatrices.jl | 136 ++++++++-------------- 1 file changed, 51 insertions(+), 85 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index af55b2986..49e823d85 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -192,12 +192,12 @@ function ParameterTable(ram_matrices::RAMMatrices) # constants for c in ram_matrices.constants - push!(partable, get_partable_row(c, position_names)) + push!(partable, partable_row(c, position_names)) end # parameters for (i, par) in enumerate(ram_matrices.parameters) - push_partable_rows!( + append_partable_rows!( partable, position_names, par, i, ram_matrices.A_ind, @@ -241,113 +241,79 @@ function parameters(partable::Union{EnsembleParameterTable, ParameterTable}) return parameters end -function get_partable_row(c::RAMConstant, position_names) - # variable names - from = position_names[c.index[2]] - to = position_names[c.index[1]] - # parameter type - if c.matrix == :A - parameter_type = :→ - elseif c.matrix == :S - parameter_type = :↔ - elseif c.matrix == :M - parameter_type = :→ - end - free = false - value_fixed = c.value - start = 0.0 - estimate = 0.0 - identifier = :const - return Dict( - :from => from, - :parameter_type => parameter_type, - :to => to, - :free => free, - :value_fixed => value_fixed, - :start => start, - :estimate => estimate, - :identifier => identifier) -end - -function cartesian_is_known(index, known_indices) - known = false - for k_in in known_indices - if (index == k_in) | ((index[1] == k_in[2]) & (index[2] == k_in[1])) - known = true - end +function matrix_to_parameter_type(matrix::Symbol) + if matrix == :A + return :→ + elseif matrix == :S + return :↔ + elseif matrix == :M + return :→ + else + throw(ArgumentError("Unsupported matrix $matrix, supported matrices are :A, :S and :M")) end - return known end -cartesian_is_known(index, known_indices::Nothing) = false +partable_row(c::RAMConstant, position_names::AbstractDict) = ( + from = position_names[c.index[2]], + parameter_type = matrix_to_parameter_type(c.matrix), + to = position_names[c.index[1]], + free = false, + value_fixed = c.value, + start = 0.0, + estimate = 0.0, + identifier = :const + ) -function get_partable_row(par, position_names, index, matrix, n_nod, known_indices) +function partable_row(par::Symbol, position_names::AbstractDict, + index::Integer, matrix::Symbol, n_nod::Integer) # variable names if matrix == :M from = Symbol("1") to = position_names[index] else - index = linear2cartesian(index, (n_nod, n_nod)) + cart_index = linear2cartesian(index, (n_nod, n_nod)) - if (matrix == :S) & (cartesian_is_known(index, known_indices)) - return nothing - elseif matrix == :S - push!(known_indices, index) - end - - from = position_names[index[2]] - to = position_names[index[1]] - end - - # parameter type - if matrix == :A - parameter_type = :→ - elseif matrix == :S - parameter_type = :↔ - elseif matrix == :M - parameter_type = :→ + from = position_names[cart_index[2]] + to = position_names[cart_index[1]] end - free = true - value_fixed = 0.0 - start = 0.0 - estimate = 0.0 - identifier = par - - return Dict( - :from => from, - :parameter_type => parameter_type, - :to => to, - :free => free, - :value_fixed => value_fixed, - :start => start, - :estimate => estimate, - :identifier => identifier) + return ( + from = from, + parameter_type = matrix_to_parameter_type(matrix), + to = to, + free = true, + value_fixed = 0.0, + start = 0.0, + estimate = 0.0, + identifier = par) end -function push_partable_rows!(partable, position_names, par, i, A_ind, S_ind, M_ind, n_nod) - A_ind = A_ind[i] - S_ind = S_ind[i] - isnothing(M_ind) || (M_ind = M_ind[i]) - - for ind in A_ind - push!(partable, get_partable_row(par, position_names, ind, :A, n_nod, nothing)) +function append_partable_rows!(partable::ParameterTable, + position_names, par::Symbol, par_index::Integer, + A_ind, S_ind, M_ind, n_nod::Integer) + for ind in A_ind[par_index] + push!(partable, partable_row(par, position_names, ind, :A, n_nod)) end - known_indices = Vector{CartesianIndex}() - for ind in S_ind - push!(partable, get_partable_row(par, position_names, ind, :S, n_nod, known_indices)) + visited_S_indices = Set{Int}() + for ind in S_ind[par_index] + if ind ∉ visited_S_indices + push!(partable, partable_row(par, position_names, ind, :S, n_nod)) + # mark index and its symmetric as visited + push!(visited_S_indices, ind) + cart_index = linear2cartesian(ind, (n_nod, n_nod)) + push!(visited_S_indices, cartesian2linear(CartesianIndex(cart_index[2], cart_index[1]), (n_nod, n_nod))) + end end if !isnothing(M_ind) - for ind in M_ind - push!(partable, get_partable_row(par, position_names, ind, :M, n_nod, nothing)) + for ind in M_ind[par_index] + push!(partable, partable_row(par, position_names, ind, :M, n_nod)) end end return nothing - end function ==(mat1::RAMMatrices, mat2::RAMMatrices) From 07ed9a9b69d342f0f971c7a7f9d13bd1c980c7a8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:29:44 -0800 Subject: [PATCH 020/174] ParTable: full support for Iterator iface --- src/frontend/specification/ParameterTable.jl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 006d3acbd..814670d71 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -93,6 +93,15 @@ end ############################################################################################ # Iteration -------------------------------------------------------------------------------- +ParameterTableRow = @NamedTuple begin + from::Symbol + parameter_type::Symbol + to::Symbol + free::Bool + value_fixed::Any + identifier::Symbol +end + Base.getindex(partable::ParameterTable, i::Integer) = (from = partable.columns.from[i], parameter_type = partable.columns.parameter_type[i], @@ -103,6 +112,11 @@ Base.getindex(partable::ParameterTable, i::Integer) = ) Base.length(partable::ParameterTable) = length(first(partable.columns)) +Base.eachindex(partable::ParameterTable) = Base.OneTo(length(partable)) + +Base.eltype(::Type{<:ParameterTable}) = ParameterTableRow +Base.iterate(partable::ParameterTable) = iterate(partable, 1) +Base.iterate(partable::ParameterTable, i::Integer) = i > length(partable) ? nothing : (partable[i], i + 1) # Sorting ---------------------------------------------------------------------------------- From c6eb0135c27548ffb019db0eb077666c53e8ee47 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:30:46 -0800 Subject: [PATCH 021/174] use Fix1 instead of anonymous function --- src/frontend/fit/standard_errors/hessian.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 2afb08822..c7d304aeb 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -18,7 +18,7 @@ function se_hessian(sem_fit::SemFit; hessian = :finitediff) hessian!(H, sem_fit.model, sem_fit.solution) elseif hessian == :finitediff H = FiniteDiff.finite_difference_hessian( - x -> objective!(sem_fit.model, x), + Base.Fix1(objective!, sem_fit.model), sem_fit.solution ) elseif hessian == :optimizer From 165c6404f240297e3b108a96157f998a01dd869a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:30:57 -0800 Subject: [PATCH 022/174] fix typo --- src/frontend/fit/standard_errors/hessian.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index c7d304aeb..0603b3ad4 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -26,7 +26,7 @@ function se_hessian(sem_fit::SemFit; hessian = :finitediff) elseif hessian == :expected throw(ArgumentError("standard errors based on the expected hessian are not implemented yet")) else - throw(ArgumentError("I dont know how to compute `$hessian` standard-errors")) + throw(ArgumentError("I don't know how to compute `$hessian` standard-errors")) end invH = c*inv(H) From b48a0c57d71413b15c6e92da082e5415fa7de95d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:31:34 -0800 Subject: [PATCH 023/174] rename vars for type stability --- src/frontend/specification/Sem.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index c91d8c677..04fadfddb 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -9,11 +9,11 @@ function Sem(; optimizer::D = SemOptimizerOptim, kwargs...) where {O, I, L, D} - kwargs = Dict{Symbol, Any}(kwargs...) + kwdict = Dict{Symbol, Any}(kwargs...) - set_field_type_kwargs!(kwargs, observed, imply, loss, optimizer, O, I, D) - - observed, imply, loss, optimizer = get_fields!(kwargs, observed, imply, loss, optimizer) + set_field_type_kwargs!(kwdict, observed, imply, loss, optimizer, O, I, D) + + observed, imply, loss, optimizer = get_fields!(kwdict, observed, imply, loss, optimizer) sem = Sem(observed, imply, loss, optimizer) @@ -27,11 +27,11 @@ function SemFiniteDiff(; optimizer::D = SemOptimizerOptim, kwargs...) where {O, I, L, D} - kwargs = Dict{Symbol, Any}(kwargs...) + kwdict = Dict{Symbol, Any}(kwargs...) + + set_field_type_kwargs!(kwdict, observed, imply, loss, optimizer, O, I, D) - set_field_type_kwargs!(kwargs, observed, imply, loss, optimizer, O, I, D) - - observed, imply, loss, optimizer = get_fields!(kwargs, observed, imply, loss, optimizer) + observed, imply, loss, optimizer = get_fields!(kwdict, observed, imply, loss, optimizer) sem = SemFiniteDiff(observed, imply, loss, optimizer) From 0391a5e506a905ae8571c3a038ba0b1ea45e6719 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:37:09 -0800 Subject: [PATCH 024/174] obj!()/grad!(): avoid tmp array creation --- src/imply/RAM/generic.jl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 3ac3c86b6..978a7b94d 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -216,7 +216,11 @@ function objective!(imply::RAM, parameters, model, has_meanstructure::Val{T}) wh imply.M_indices, parameters) - imply.I_A .= I - imply.A + @inbounds for (j, I_Aj, Aj) in zip(axes(imply.A, 2), eachcol(imply.I_A), eachcol(imply.A)) + for i in axes(imply.A, 1) + I_Aj[i] = ifelse(i == j, 1, 0) - Aj[i] + end + end copyto!(imply.F⨉I_A⁻¹, imply.F) rdiv!(imply.F⨉I_A⁻¹, factorize(imply.I_A)) @@ -244,8 +248,11 @@ function gradient!(imply::RAM, parameters, model::AbstractSemSingle, has_meanstr imply.M_indices, parameters) - imply.I_A .= I - imply.A - copyto!(imply.I_A⁻¹, imply.I_A) + @inbounds for (j, I_Aj, Aj) in zip(axes(imply.A, 2), eachcol(imply.I_A), eachcol(imply.A)) + for i in axes(imply.A, 1) + I_Aj[i] = ifelse(i == j, 1, 0) - Aj[i] + end + end imply.I_A⁻¹ .= LinearAlgebra.inv!(factorize(imply.I_A⁻¹)) imply.F⨉I_A⁻¹ .= imply.F*imply.I_A⁻¹ From b18e658855e67084f2ab0b565597154981f272ec Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:38:07 -0800 Subject: [PATCH 025/174] grad!(): avoid extra array copying --- src/imply/RAM/generic.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 978a7b94d..c3fae1c21 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -254,8 +254,8 @@ function gradient!(imply::RAM, parameters, model::AbstractSemSingle, has_meanstr end end - imply.I_A⁻¹ .= LinearAlgebra.inv!(factorize(imply.I_A⁻¹)) - imply.F⨉I_A⁻¹ .= imply.F*imply.I_A⁻¹ + imply.I_A⁻¹ = LinearAlgebra.inv!(factorize(imply.I_A)) + mul!(imply.F⨉I_A⁻¹, imply.F, imply.I_A⁻¹) Σ_RAM!( imply.Σ, From 764d4a0df4fe3d0c1e109fed4c8d1d365f6f59a9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:38:33 -0800 Subject: [PATCH 026/174] check_acyclic(): use istril/u() --- src/imply/RAM/generic.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index c3fae1c21..5912d9baa 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -351,9 +351,9 @@ function check_acyclic(A_pre, n_par, A_indices) acyclic = isone(det(I-A_rand)) # check if A is lower or upper triangular - if iszero(A_rand[.!tril(ones(Bool, size(A_pre)...))]) + if istril(A_rand) A_pre = LowerTriangular(A_pre) - elseif iszero(A_rand[.!tril(ones(Bool, size(A_pre)...))']) + elseif istriu(A_rand) A_pre = UpperTriangular(A_pre) elseif acyclic @info "Your model is acyclic, specifying the A Matrix as either Upper or Lower Triangular can have great performance benefits.\n" maxlog=1 From 0bbf5c9f5f8a1a015b66a637f2e2d2282d3f5f58 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 15:40:57 -0800 Subject: [PATCH 027/174] grad!(SemML): reduce * ops --- src/loss/ML/ML.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 1902e9459..0ab509123 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -449,7 +449,7 @@ function gradient!(semml::SemML, par, model::AbstractSemSingle, has_meanstructur μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ - gradient += -2k*∇M - 2vec(k'M'I_A⁻¹')'∇A - 2vec(k'k*S*I_A⁻¹')'∇A - vec(k'k)'∇S + gradient += -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S end return gradient' @@ -490,7 +490,7 @@ function objective_gradient!( μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ - gradient += -2k*∇M - 2vec(k'M'I_A⁻¹')'∇A - 2vec(k'k*S*I_A⁻¹')'∇A - vec(k'k)'∇S + gradient += -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S end return objective, gradient' From 74e23e966db844bbcd991717fda22c5c834e3f26 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 16:03:22 -0800 Subject: [PATCH 028/174] obj/grad/hess!(SemML): avoid extra arr copying --- src/loss/ML/ML.jl | 65 ++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 0ab509123..377fdfd4d 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -87,11 +87,9 @@ function objective!( copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) return non_posdef_return(par) end - + isposdef(Σ_chol) || return non_posdef_return(par) ld = logdet(Σ_chol) - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) #mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) if T @@ -116,12 +114,8 @@ function gradient!( copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) - return ones(eltype(par), size(par)) - end - - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + isposdef(Σ_chol) || return ones(eltype(par), size(par)) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) if T @@ -149,12 +143,9 @@ function hessian!( copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) + isposdef(Σ_chol) || return diagm(fill(one(eltype(par)), length(par))) - end - - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) if semml.approximate_hessian hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ @@ -196,12 +187,11 @@ function objective_gradient!( copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) + if !isposdef(Σ_chol) return non_posdef_return(par), ones(eltype(par), size(par)) else ld = logdet(Σ_chol) - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) if T @@ -233,12 +223,11 @@ function objective_hessian!( copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) + if !isposdef(Σ_chol) return non_posdef_return(par), diagm(fill(one(eltype(par)), length(par))) else ld = logdet(Σ_chol) - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) objective = ld + tr(Σ⁻¹Σₒ) @@ -282,12 +271,9 @@ function gradient_hessian!( copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) + isposdef(Σ_chol) || return ones(eltype(par), size(par)), diagm(fill(one(eltype(par)), length(par))) - end - - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ @@ -332,16 +318,14 @@ function objective_gradient_hessian!( copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) + if !isposdef(Σ_chol) objective = non_posdef_return(par) gradient = ones(eltype(par), size(par)) hessian = diagm(fill(one(eltype(par)), length(par))) return objective, gradient, hessian end - ld = logdet(Σ_chol) - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) objective = ld + tr(Σ⁻¹Σₒ) @@ -409,11 +393,9 @@ function objective!( copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) return non_posdef_return(par) end - + isposdef(Σ_chol) || return non_posdef_return(par) ld = logdet(Σ_chol) - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) if T @@ -432,13 +414,11 @@ function gradient!(semml::SemML, par, model::AbstractSemSingle, has_meanstructur F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), I_A⁻¹ = I_A⁻¹(imply(model)), ∇A = ∇A(imply(model)), ∇S = ∇S(imply(model)), ∇M = ∇M(imply(model)), μ = μ(imply(model)), μₒ = obs_mean(observed(model)) - + copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) return ones(eltype(par), size(par)) end - - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + isposdef(Σ_chol) || return ones(eltype(par), size(par)) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) #mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) C = F⨉I_A⁻¹'*(I-Σₒ*Σ⁻¹)'*Σ⁻¹*F⨉I_A⁻¹ @@ -467,17 +447,16 @@ function objective_gradient!( S = S(imply(model)), M = M(imply(model)), F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), I_A⁻¹ = I_A⁻¹(imply(model)), ∇A = ∇A(imply(model)), ∇S = ∇S(imply(model)), ∇M = ∇M(imply(model)), μ = μ(imply(model)), μₒ = obs_mean(observed(model)) - + copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - - if !isposdef(Σ_chol) + if !isposdef(Σ_chol) objective = non_posdef_return(par) gradient = ones(eltype(par), size(par)) return objective, gradient else ld = logdet(Σ_chol) - Σ⁻¹ .= LinearAlgebra.inv!(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) objective = ld + tr(Σ⁻¹Σₒ) From 67a988e6478c9538dc81255b8165f68fa3158023 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 17:51:50 -0800 Subject: [PATCH 029/174] use .+= to reduce allocs --- src/loss/ML/ML.jl | 12 ++++++------ src/loss/WLS/WLS.jl | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 377fdfd4d..4a47f7847 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -158,7 +158,7 @@ function hessian!( # outer H_outer = 2*kron(Σ⁻¹ΣₒΣ⁻¹, Σ⁻¹) - kron(Σ⁻¹, Σ⁻¹) hessian = ∇Σ'*H_outer*∇Σ - hessian += ∇²Σ + hessian .+= ∇²Σ end return hessian @@ -241,7 +241,7 @@ function objective_hessian!( # outer H_outer = 2*kron(Σ⁻¹ΣₒΣ⁻¹, Σ⁻¹) - kron(Σ⁻¹, Σ⁻¹) hessian = ∇Σ'*H_outer*∇Σ - hessian += ∇²Σ + hessian .+= ∇²Σ end return objective, hessian @@ -289,7 +289,7 @@ function gradient_hessian!( # outer H_outer = 2*kron(Σ⁻¹ΣₒΣ⁻¹, Σ⁻¹) - kron(Σ⁻¹, Σ⁻¹) hessian = ∇Σ'*H_outer*∇Σ - hessian += ∇²Σ + hessian .+= ∇²Σ end return gradient', hessian @@ -343,7 +343,7 @@ function objective_gradient_hessian!( # outer H_outer = 2*kron(Σ⁻¹ΣₒΣ⁻¹, Σ⁻¹) - kron(Σ⁻¹, Σ⁻¹) hessian = ∇Σ'*H_outer*∇Σ - hessian += ∇²Σ + hessian .+= ∇²Σ end return objective, gradient', hessian @@ -429,7 +429,7 @@ function gradient!(semml::SemML, par, model::AbstractSemSingle, has_meanstructur μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ - gradient += -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S + gradient .+= -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S end return gradient' @@ -469,7 +469,7 @@ function objective_gradient!( μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ - gradient += -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S + gradient .+= -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S end return objective, gradient' diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 2fd053f2d..eae2b5381 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -150,7 +150,7 @@ function hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructu if !semwls.approximate_hessian J = -2*(σ₋'*semwls.V)' ∇²Σ_function!(∇²Σ, J, par) - hessian += ∇²Σ + hessian .+= ∇²Σ end return hessian end @@ -192,7 +192,7 @@ function objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_m if !semwls.approximate_hessian J = -2*(σ₋'*semwls.V)' ∇²Σ_function!(∇²Σ, J, par) - hessian += ∇²Σ + hessian .+= ∇²Σ end return objective, hessian @@ -216,7 +216,7 @@ function gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_me if !semwls.approximate_hessian J = -2*(σ₋'*semwls.V)' ∇²Σ_function!(∇²Σ, J, par) - hessian += ∇²Σ + hessian .+= ∇²Σ end return gradient, hessian @@ -240,7 +240,7 @@ function objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSing if !semwls.approximate_hessian J = -2*(σ₋'*semwls.V)' ∇²Σ_function!(∇²Σ, J, par) - hessian += ∇²Σ + hessian .+= ∇²Σ end return objective, gradient, hessian end From 28950f3792e6084b00782109c7eb4445fcb16953 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 18:43:48 -0700 Subject: [PATCH 030/174] ParTable: update StenGraph-based ctor * use graph as a main parameter * simplify rows processing * don't reallocate table.columns --- src/frontend/specification/StenoGraphs.jl | 99 ++++++++----------- .../political_democracy.jl | 11 +-- 2 files changed, 48 insertions(+), 62 deletions(-) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index a9b655477..c3ddb611e 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -4,6 +4,8 @@ ### Define Modifiers ############################################################################################ +AbstractStenoGraph = AbstractVector + # fixed parameter values struct Fixed{N} <: EdgeModifier value::N @@ -28,91 +30,76 @@ label(args...) = Label(args) ### constructor for parameter table from graph ############################################################################################ -function ParameterTable(;graph, observed_vars, latent_vars, g = 1, parname = :θ) +function ParameterTable(graph::AbstractStenoGraph; + observed_vars, latent_vars, + group::Integer = 1, param_prefix = :θ) graph = unique(graph) n = length(graph) - from = Vector{Symbol}(undef, n) - parameter_type = Vector{Symbol}(undef, n) - to = Vector{Symbol}(undef, n) - free = ones(Bool, n) - value_fixed = zeros(n) - start = zeros(n) - estimate = zeros(n) - identifier = Vector{Symbol}(undef, n); identifier .= Symbol("") + + partable = ParameterTable( + latent_vars = latent_vars, + observed_vars = observed_vars) + from = resize!(partable.columns.from, n) + parameter_type = resize!(partable.columns.parameter_type, n) + to = resize!(partable.columns.to, n) + free = fill!(resize!(partable.columns.free, n), true) + value_fixed = fill!(resize!(partable.columns.value_fixed, n), NaN) + start = fill!(resize!(partable.columns.start, n), NaN) + identifier = fill!(resize!(partable.columns.identifier, n), Symbol("")) # group = Vector{Symbol}(undef, n) # start_partable = zeros(Bool, n) - sorted_vars = Vector{Symbol}() - for (i, element) in enumerate(graph) - if element isa DirectedEdge - from[i] = element.src.node - to[i] = element.dst.node + edge = element isa ModifiedEdge ? element.edge : element + from[i] = edge.src.node + to[i] = edge.dst.node + if edge isa DirectedEdge parameter_type[i] = :→ - elseif element isa UndirectedEdge - from[i] = element.src.node - to[i] = element.dst.node + elseif edge isa UndirectedEdge parameter_type[i] = :↔ - elseif element isa ModifiedEdge - if element.edge isa DirectedEdge - from[i] = element.edge.src.node - to[i] = element.edge.dst.node - parameter_type[i] = :→ - elseif element.edge isa UndirectedEdge - from[i] = element.edge.src.node - to[i] = element.edge.dst.node - parameter_type[i] = :↔ - end + else + throw(ArgumentError("The graph contains an unsupported edge of type $(typeof(edge)).")) + end + if element isa ModifiedEdge for modifier in values(element.modifiers) + modval = modifier.value[group] if modifier isa Fixed - if modifier.value[g] == :NaN + if modval == :NaN free[i] = true value_fixed[i] = 0.0 else free[i] = false - value_fixed[i] = modifier.value[g] + value_fixed[i] = modval end elseif modifier isa Start - start_partable[i] = modifier.value[g] == :NaN - start[i] = modifier.value[g] + start_partable[i] = modval == :NaN + start[i] = modval elseif modifier isa Label - if modifier.value[g] == :NaN + if modval == :NaN throw(DomainError(NaN, "NaN is not allowed as a parameter label.")) end - identifier[i] = modifier.value[g] + identifier[i] = modval end end - end + end end # make identifiers for parameters that are not labeled current_id = 1 for i in 1:length(identifier) - if (identifier[i] == Symbol("")) & free[i] - identifier[i] = Symbol(parname, :_, current_id) - current_id += 1 - elseif (identifier[i] == Symbol("")) & !free[i] - identifier[i] = :const - elseif (identifier[i] != Symbol("")) & !free[i] - @warn "You labeled a constant. Please check if the labels of your graph are correct." + if identifier[i] == Symbol("") + if free[i] + identifier[i] = Symbol(param_prefix, :_, current_id) + current_id += 1 + else + identifier[i] = :const + end + elseif !free[i] + @warn "You labeled a constant ($(identifier[i])=$(value_fixed[i])). Please check if the labels of your graph are correct." end end - return StructuralEquationModels.ParameterTable( - Dict( - :from => from, - :parameter_type => parameter_type, - :to => to, - :free => free, - :value_fixed => value_fixed, - :start => start, - :estimate => estimate, - :identifier => identifier), - Dict( - :latent_vars => latent_vars, - :observed_vars => observed_vars, - :sorted_vars => sorted_vars) - ) + return partable end ############################################################################################ diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index ed70bdb3d..6e7162173 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -94,6 +94,7 @@ start_test_mean = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fi semoptimizer = SemOptimizerOptim @testset "RAMMatrices | constructor | Optim" begin include("constructor.jl") end + semoptimizer = SemOptimizerNLopt @testset "RAMMatrices | constructor | NLopt" begin include("constructor.jl") end @@ -154,10 +155,9 @@ graph = @StenoGraph begin y8 ↔ y4 + y6 end -spec = ParameterTable( +spec = ParameterTable(graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) sort!(spec) @@ -188,10 +188,9 @@ graph = @StenoGraph begin Symbol("1") → fixed(0)*ind60 end -spec_mean = ParameterTable( +spec_mean = ParameterTable(graph, latent_vars = latent_vars, - observed_vars = observed_vars, - graph = graph) + observed_vars = observed_vars) sort!(spec_mean) From 9b51e599b989fab39430f4615dc6a717b7fff711 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 11 Mar 2024 17:52:11 -0700 Subject: [PATCH 031/174] EnsParTable: update dict-based and graph-based ctors * don't use keywords for main params as it complicates dispatch --- .../specification/EnsembleParameterTable.jl | 15 ++++------- src/frontend/specification/StenoGraphs.jl | 26 +++++++------------ test/examples/multigroup/multigroup.jl | 5 ++-- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index f2a38c56d..c3d5929e3 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -2,7 +2,7 @@ ### Types ############################################################################################ -mutable struct EnsembleParameterTable{C} <: AbstractParameterTable +mutable struct EnsembleParameterTable{C <: AbstractDict{<:Any, ParameterTable}} <: AbstractParameterTable tables::C end @@ -48,15 +48,10 @@ end =# ### get parameter table from RAMMatrices ############################################################################################ -function EnsembleParameterTable(args...; groups) - partable = EnsembleParameterTable(nothing) - - for (group, ram_matrices) in zip(groups, args) - push!(partable.tables, group => ParameterTable(ram_matrices)) - end - - return partable -end +EnsembleParameterTable(spec_ensemble::AbstractDict{K}) where K = + EnsembleParameterTable(Dict{K, ParameterTable}( + group => convert(ParameterTable, spec) + for (group, spec) in pairs(spec_ensemble))) ############################################################################################ ### Pretty Printing diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index c3ddb611e..3f360f3b7 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -106,23 +106,17 @@ end ### constructor for EnsembleParameterTable from graph ############################################################################################ -function EnsembleParameterTable(;graph, observed_vars, latent_vars, groups) +function EnsembleParameterTable(graph::AbstractStenoGraph; + observed_vars, latent_vars, groups) graph = unique(graph) - partable = EnsembleParameterTable(nothing) - - for (i, group) in enumerate(groups) - push!( - partable.tables, - Symbol(group) => - ParameterTable(; - graph = graph, - observed_vars = observed_vars, - latent_vars = latent_vars, - g = i, - parname = Symbol(:g, i))) - end - - return partable + partables = Dict(group => ParameterTable( + graph; + observed_vars = observed_vars, + latent_vars = latent_vars, + group = i, + param_prefix = Symbol(:g, group)) + for (i, group) in enumerate(groups)) + return EnsembleParameterTable(partables) end \ No newline at end of file diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index fc40e36d4..5a34c4d86 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -58,9 +58,8 @@ specification_g2 = RAMMatrices(; colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed]) partable = EnsembleParameterTable( - specification_g1, - specification_g2; - groups = [:Pasteur, :Grant_White] + Dict(:Pasteur => specification_g1, + :Grant_White => specification_g2) ) specification_miss_g1 = nothing From e6fde8c15022091545c5fe3a1a537cc6228bf501 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 18:39:48 -0800 Subject: [PATCH 032/174] start_fabin3: optimize indexing * use sorted searches * use dict to map indicator to param index --- .../start_val/start_fabin3.jl | 113 +++++++++--------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 25d3e525b..348f419bb 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -66,10 +66,8 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) C_indices = CartesianIndices((n_nod, n_nod)) # check in which matrix each parameter appears - - indices = Vector{CartesianIndex}(undef, n_par) - start_val = zeros(n_par) + indices = Vector{CartesianIndex{2}}(undef, n_par) #= in_S = length.(S_ind) .!= 0 in_A = length.(A_ind) .!= 0 @@ -89,16 +87,12 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) # set undirected parameters in S for (i, S_ind) in enumerate(S_ind) - if length(S_ind) != 0 - c_ind = C_indices[S_ind][1] - if c_ind[1] == c_ind[2] - if c_ind[1] ∈ F_ind - index_position = findall(c_ind[1] .== F_ind) - start_val[i] = Σ[index_position[1], index_position[1]]/2 - else - start_val[i] = 0.05 - end - end # covariances stay 0 + for c_ind in C_indices[S_ind] + (c_ind[1] == c_ind[2]) || continue # covariances stay 0 + pos = searchsortedfirst(F_ind, c_ind[1]) + start_val[i] = (pos <= length(F_ind)) && (F_ind[pos] == c_ind[1]) ? + Σ[pos, pos]/2 : 0.05 + break # i-th parameter initialized end end @@ -107,76 +101,78 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) A_ind_c = [linear2cartesian(ind, (n_nod, n_nod)) for ind in A_ind] # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c]) - for i ∈ findall(.!(1:n_nod .∈ [F_ind])) + for i ∈ setdiff(1:n_nod, F_ind) reference = Int64[] indicators = Int64[] - loadings = Symbol[] - - for (j, ind_c_vec) in enumerate(A_ind_c) - for ind_c in ind_c_vec - if (ind_c[2] == i) & (ind_c[1] ∈ F_ind) - push!(indicators, ind_c[1]) - push!(loadings, parameters[j]) + indicator2parampos = Dict{Int, Int}() + + for (j, Aj_ind_c) in enumerate(A_ind_c) + for ind_c in Aj_ind_c + (ind_c[2] == i) || continue + ind_pos = searchsortedfirst(F_ind, ind_c[1]) + if (ind_pos <= length(F_ind)) && (F_ind[ind_pos] == ind_c[1]) + push!(indicators, ind_pos) + indicator2parampos[ind_pos] = j end end end - for ram_constant in constants - if (ram_constant.matrix == :A) && (ram_constant.index[2] == i) && (ram_constant.index[1] ∈ F_ind) - push!(loadings, Symbol("")) - if isone(ram_constant.value) - push!(reference, ram_constant.index[1]) - else - push!(indicators, ram_constant.index[1]) + for ram_const in constants + if (ram_const.matrix == :A) && (ram_const.index[2] == i) + ind_pos = searchsortedfirst(F_ind, ram_const.index[1]) + if (ind_pos <= length(F_ind)) && (F_ind[ind_pos] == ram_const.index[1]) + if isone(ram_const.value) + push!(reference, ind_pos) + else + push!(indicators, ind_pos) + # no parameter associated + end end end end - reference = [findfirst(x -> x == ref, F_ind) for ref in reference] - indicators = [findfirst(x -> x == ind, F_ind) for ind in indicators] - # is there at least one reference indicator? - if size(reference, 1) > 0 - if size(reference, 1) > 1 - @warn "You have more than 1 scaling indicator" - reference = reference[1] + if length(reference) > 0 + if length(reference) > 1 + if isempty(indicator2parampos) # don't warn if entire column is fixed + @warn "You have more than 1 scaling indicator for $(ram_matrices.colnames[i])" + end + ref = reference[1] else - reference = reference[1] + ref = reference[1] end for (j, indicator) in enumerate(indicators) - if indicator != reference - instruments = indicators[.!(indicators .∈ [[reference; indicator]])] + if indicator != ref + instruments = filter(i -> (i != ref) && (i != indicator), indicators) s32 = Σ[instruments, indicator] - s13 = Σ[reference, instruments] + s13 = Σ[ref, instruments] S33 = Σ[instruments, instruments] - + if size(instruments, 1) == 1 temp = S33[1]/s13[1] λ = s32[1]*temp/(s13[1]*temp) - start_val[isequal.(parameters, loadings[j])] .= λ else temp = S33\s13 λ = s32'*temp/(s13'*temp) - start_val[isequal.(parameters, loadings[j])] .= λ end - + start_val[indicator2parampos[indicator]] = λ end end # no reference indicator: - else - reference = indicators[1] - λ = zeros(size(indicators, 1)); λ[1] = 1.0 + elseif length(indicators) > 0 + ref = indicators[1] + λ = Vector{Float64}(undef, length(indicators)); λ[1] = 1.0 for (j, indicator) in enumerate(indicators) - if indicator != reference - instruments = indicators[.!(indicators .∈ [[reference; indicator]])] + if indicator != ref + instruments = filter(i -> (i != ref) && (i != indicator), indicators) s32 = Σ[instruments, indicator] - s13 = Σ[reference, instruments] + s13 = Σ[ref, instruments] S33 = Σ[instruments, instruments] - if size(instruments, 1) == 1 + if length(instruments) == 1 temp = S33[1]/s13[1] λ[j] = s32[1]*temp/(s13[1]*temp) else @@ -199,19 +195,23 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) λ = λ .* sign(Ψ) .* sqrt(abs(Ψ)) for (j, indicator) ∈ enumerate(indicators) - start_val[isequal.(parameters, loadings[j])] .= λ[j] + if (parampos = get(indicator2parampos, indicator, 0)) != 0 + start_val[parampos] = λ[j] + end end + else + @warn "No scaling indicators for $(ram_matrices.colnames[i])" end end # set means if !isnothing(M_ind) for (i, M_ind) in enumerate(M_ind) - if length(M_ind) != 0 + if length(M_ind) != 0 ind = M_ind[1] - if ind[1] ∈ F_ind - index_position = findfirst(ind[1] .== F_ind) - start_val[i] = μ[index_position] + pos = searchsortedfirst(F_ind, ind[1]) + if (pos <= length(F_ind)) && (F_ind[pos] == ind[1]) + start_val[i] = μ[pos] end # latent means stay 0 end end @@ -221,6 +221,5 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end function is_in_Λ(ind_vec, F_ind) - res = [!(ind[2] ∈ F_ind) & (ind[1] ∈ F_ind) for ind in ind_vec] - return any(res) + return any(ind -> !(ind[2] ∈ F_ind) && (ind[1] ∈ F_ind), ind_vec) end \ No newline at end of file From 4c61f105cfd0524ea85c39c7fd4fdde3e073a8aa Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 18:45:22 -0800 Subject: [PATCH 033/174] start_fabin3(): optimize math * extract common code to calculate_lambda() * use 3-arg dot() * use in-place ops --- .../start_val/start_fabin3.jl | 60 ++++++++----------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 348f419bb..74877c3c0 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -101,6 +101,22 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) A_ind_c = [linear2cartesian(ind, (n_nod, n_nod)) for ind in A_ind] # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c]) + function calculate_lambda(ref::Integer, indicator::Integer, + indicators::AbstractVector{<:Integer}) + instruments = filter(i -> (i != ref) && (i != indicator), indicators) + if length(instruments) == 1 + s13 = Σ[ref, instruments[1]] + s32 = Σ[instruments[1], indicator] + return s32/s13 + else + s13 = Σ[ref, instruments] + s32 = Σ[instruments, indicator] + S33 = Σ[instruments, instruments] + temp = S33\s13 + return dot(s32, temp)/dot(s13, temp) + end + end + for i ∈ setdiff(1:n_nod, F_ind) reference = Int64[] indicators = Int64[] @@ -143,21 +159,8 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end for (j, indicator) in enumerate(indicators) - if indicator != ref - instruments = filter(i -> (i != ref) && (i != indicator), indicators) - - s32 = Σ[instruments, indicator] - s13 = Σ[ref, instruments] - S33 = Σ[instruments, instruments] - - if size(instruments, 1) == 1 - temp = S33[1]/s13[1] - λ = s32[1]*temp/(s13[1]*temp) - else - temp = S33\s13 - λ = s32'*temp/(s13'*temp) - end - start_val[indicator2parampos[indicator]] = λ + if (indicator != ref) && (parampos = get(indicator2parampos, indicator, 0)) != 0 + start_val[parampos] = calculate_lambda(ref, indicator, indicators) end end # no reference indicator: @@ -166,33 +169,20 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) λ = Vector{Float64}(undef, length(indicators)); λ[1] = 1.0 for (j, indicator) in enumerate(indicators) if indicator != ref - instruments = filter(i -> (i != ref) && (i != indicator), indicators) - - s32 = Σ[instruments, indicator] - s13 = Σ[ref, instruments] - S33 = Σ[instruments, instruments] - - if length(instruments) == 1 - temp = S33[1]/s13[1] - λ[j] = s32[1]*temp/(s13[1]*temp) - else - temp = S33\s13 - λ[j] = s32'*temp/(s13'*temp) - end - + λ[j] = calculate_lambda(ref, indicator, indicators) end end Σ_λ = Σ[indicators, indicators] - D = λ*λ' ./ sum(λ.^2) + l₂ = sum(abs2, λ) + D = λ*λ' ./ l₂ θ = (I - D.^2)\(diag(Σ_λ - D*Σ_λ*D)) - + # 3. psi Σ₁ = Σ_λ - Diagonal(θ) - l₂ = sum(λ.^2) - Ψ = sum( sum(λ.*Σ₁, dims = 1).*λ') ./ l₂^2 - - λ = λ .* sign(Ψ) .* sqrt(abs(Ψ)) + Ψ = dot(λ, Σ₁, λ) / l₂^2 + + λ .*= sign(Ψ) * sqrt(abs(Ψ)) for (j, indicator) ∈ enumerate(indicators) if (parampos = get(indicator2parampos, indicator, 0)) != 0 From c6641d499d2365016dccfdfe622922f095f6172d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 11 Mar 2024 22:36:22 -0700 Subject: [PATCH 034/174] start_fabin3(): directly access imply.ram_matrices --- src/additional_functions/start_val/start_fabin3.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 74877c3c0..fdc32a6bd 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -25,7 +25,7 @@ function start_fabin3( args...; kwargs...) return start_fabin3( - ram_matrices(imply), + imply.ram_matrices, obs_cov(observed), obs_mean(observed)) end @@ -43,7 +43,7 @@ function start_fabin3( end return start_fabin3( - ram_matrices(imply), + imply.ram_matrices, observed.em_model.Σ, observed.em_model.μ) end From 2c9ffc9c08214ab12d52aaa65aec0d282ddfc78a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 18:47:13 -0800 Subject: [PATCH 035/174] reorder_data(): optimize use dict for faster observed-to-specified matches --- src/observed/data.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/observed/data.jl b/src/observed/data.jl index 79468e473..fe2f37dc9 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -151,8 +151,8 @@ function reorder_data(data::AbstractArray, spec_colnames, obs_colnames) if spec_colnames == obs_colnames return data else - new_position = [findall(x .== obs_colnames)[1] for x in spec_colnames] - data = data[:, new_position] - return data + obs_positions = Dict(col => i for (i, col) in enumerate(obs_colnames)) + new_positions = [obs_positions[col] for col in spec_colnames] + return data[:, new_positions] end end \ No newline at end of file From 0aa69031dc1a6c4b4bf5c915534b242272248bb4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 9 Mar 2024 18:47:51 -0800 Subject: [PATCH 036/174] SemObservedData(): cleanup code --- src/observed/data.jl | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/src/observed/data.jl b/src/observed/data.jl index fe2f37dc9..419a55cf2 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -102,28 +102,12 @@ function SemObservedData(; data = Matrix(data) end - n_obs, n_man = Float64.(size(data)) - - if compute_covariance - obs_cov = Statistics.cov(data) - else - obs_cov = nothing - end - - # if a meanstructure is needed, compute observed means - if meanstructure - obs_mean = vcat(Statistics.mean(data, dims = 1)...) - else - obs_mean = nothing - end - - if rowwise - data_rowwise = [data[i, :] for i = 1:convert(Int64, n_obs)] - else - data_rowwise = nothing - end - - return SemObservedData(data, obs_cov, obs_mean, n_man, n_obs, data_rowwise) + return SemObservedData(data, + compute_covariance ? Statistics.cov(data) : nothing, + meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, + Float64.(size(data, 2)), + Float64.(size(data, 1)), + rowwise ? [data[i, :] for i in axes(data, 1)] : nothing) end ############################################################################################ From 28965a0712ad2e65ad9899e0358056806da4f3db Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 22:04:50 -0700 Subject: [PATCH 037/174] SemSpecification base type --- src/frontend/specification/ParameterTable.jl | 2 -- src/frontend/specification/RAMMatrices.jl | 2 +- src/types.jl | 6 +++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 814670d71..690cf8266 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -1,5 +1,3 @@ -abstract type AbstractParameterTable end - ############################################################################################ ### Types ############################################################################################ diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 49e823d85..c58421071 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -49,7 +49,7 @@ end AbstractArrayParamsMap = AbstractVector{<:AbstractVector{<:Integer}} ArrayParamsMap = Vector{Vector{Int}} -struct RAMMatrices +struct RAMMatrices <: SemSpecification A_ind::ArrayParamsMap S_ind::ArrayParamsMap F_ind::Vector{Int} diff --git a/src/types.jl b/src/types.jl index e506f65e5..c4364d6b9 100644 --- a/src/types.jl +++ b/src/types.jl @@ -254,4 +254,8 @@ loss(model::AbstractSemSingle) = model.loss Returns the optimizer part of a model. """ -optimizer(model::AbstractSemSingle) = model.optimizer \ No newline at end of file +optimizer(model::AbstractSemSingle) = model.optimizer + +abstract type SemSpecification end + +abstract type AbstractParameterTable <: SemSpecification end From a8c8e8a831b6301fa7941ad13b195878c9d5e055 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:06:32 -0700 Subject: [PATCH 038/174] use SemSpecification in method signatures --- src/imply/RAM/generic.jl | 2 +- src/imply/RAM/symbolic.jl | 2 +- src/observed/covariance.jl | 2 +- src/observed/data.jl | 2 +- src/observed/missing.jl | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 5912d9baa..cf78c0944 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -100,7 +100,7 @@ using StructuralEquationModels ############################################################################################ function RAM(; - specification, + specification::SemSpecification, #vech = false, gradient = true, meanstructure = false, diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 51f8fe7e6..2c497bb1e 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -87,7 +87,7 @@ end ############################################################################################ function RAMSymbolic(; - specification, + specification::SemSpecification, loss_types = nothing, vech = false, gradient = true, diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 55016fd95..fa73f5a3e 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -47,7 +47,7 @@ struct SemObservedCovariance{B, C, D, O} <: SemObserved end using StructuralEquationModels function SemObservedCovariance(; - specification, + specification::Union{SemSpecification, Nothing} = nothing, obs_cov, obs_colnames = nothing, diff --git a/src/observed/data.jl b/src/observed/data.jl index 419a55cf2..49aa1fee4 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -56,7 +56,7 @@ end function SemObservedData(; - specification, + specification::Union{SemSpecification, Nothing}, data, obs_colnames = nothing, diff --git a/src/observed/missing.jl b/src/observed/missing.jl index adf9c2f1c..be4b49784 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -86,7 +86,7 @@ end ############################################################################################ function SemObservedMissing(; - specification, + specification::Union{SemSpecification, Nothing}, data, obs_colnames = nothing, From cdc7618807d5c817d9121801176904e587454d24 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:40:33 -0700 Subject: [PATCH 039/174] SemSpecification: vars API --- src/types.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/types.jl b/src/types.jl index c4364d6b9..a75b5b45a 100644 --- a/src/types.jl +++ b/src/types.jl @@ -258,4 +258,16 @@ optimizer(model::AbstractSemSingle) = model.optimizer abstract type SemSpecification end +# observed + latent +vars(spec::SemSpecification) = + error("vars(spec::$(typeof(spec))) is not implemented") +observed_vars(spec::SemSpecification) = + error("observed_vars(spec::$(typeof(spec))) is not implemented") +latent_vars(spec::SemSpecification) = + error("latent_vars(spec::$(typeof(spec))) is not implemented") + +nvars(spec::SemSpecification) = length(vars(spec)) +nobserved_vars(spec::SemSpecification) = length(observed_vars(spec)) +nlatent_vars(spec::SemSpecification) = length(latent_vars(spec)) + abstract type AbstractParameterTable <: SemSpecification end From 1fc18cc27fa0d5a65d20ba17b3feee40eb74a713 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:40:33 -0700 Subject: [PATCH 040/174] RAMMatrices: vars API --- src/frontend/specification/RAMMatrices.jl | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index c58421071..109c034f3 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -60,6 +60,29 @@ struct RAMMatrices <: SemSpecification size_F::Tuple{Int, Int} end +nparams(ram::RAMMatrices) = length(ram.A_ind) +nvars(ram::RAMMatrices) = ram.size_F[2] +nobserved_vars(ram::RAMMatrices) = ram.size_F[1] +nlatent_vars(ram::RAMMatrices) = nvars(ram) - nobserved_vars(ram) + +function observed_vars(ram::RAMMatrices) + if isnothing(ram.colnames) + @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" + return nothing + else + return view(ram.colnames, ram.F_ind) + end +end + +function latent_vars(ram::RAMMatrices) + if isnothing(ram.colnames) + @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" + return nothing + else + return view(ram.colnames, setdiff(eachindex(ram.colnames), ram.F_ind)) + end +end + ############################################################################################ ### Constructor ############################################################################################ From b30d2a6999ebd7ec8dc77a3abf681e244498ac8a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 17:20:36 -0700 Subject: [PATCH 041/174] ParamTable: vars API --- src/frontend/specification/ParameterTable.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 690cf8266..74114942f 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -34,6 +34,14 @@ function ParameterTable(; observed_vars::Union{AbstractVector{Symbol}, Nothing}= return ParameterTable(columns, variables) end +vars(partable::ParameterTable) = + !isempty(partable.variables.sorted) ? partable.variables.sorted : + vcat(partable.variables.observed, partable.variables.latent) +observed_vars(partable::ParameterTable) = partable.variables.observed +latent_vars(partable::ParameterTable) = partable.variables.latent + +nvars(partable::ParameterTable) = + length(partable.variables.latent) + length(partable.variables.observed) ############################################################################################ ### Convert to other types @@ -158,6 +166,7 @@ function Base.sort!(partable::ParameterTable) copyto!(resize!(partable.variables.sorted, length(sorted_vars)), sorted_vars) + @assert length(partable.variables.sorted) == nvars(partable) return partable end From dd3a97c4c29bcf591a71689130ec118348f3a862 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 23:55:42 -0700 Subject: [PATCH 042/174] RAM imply: use vars API --- src/imply/RAM/generic.jl | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index cf78c0944..dbe8637e2 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -65,7 +65,7 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{A1, A2, A3, A4, A5, A6, V, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3, B, D} <: SemImply +mutable struct RAM{A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3, B, D} <: SemImply Σ::A1 A::A2 S::A3 @@ -73,7 +73,6 @@ mutable struct RAM{A1, A2, A3, A4, A5, A6, V, V2, I1, I2, I3, M1, M2, M3, M4, S1 μ::A5 M::A6 - n_par::V ram_matrices::V2 has_meanstructure::B @@ -111,9 +110,9 @@ function RAM(; # get dimensions of the model - n_par = length(ram_matrices.parameters) - n_var, n_nod = ram_matrices.size_F - parameters = ram_matrices.parameters + n_par = nparams(ram_matrices) + n_obs = nobserved_vars(ram_matrices) + n_var = nvars(ram_matrices) F = zeros(ram_matrices.size_F); F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 # get indices @@ -122,23 +121,23 @@ function RAM(; !isnothing(ram_matrices.M_ind) ? M_indices = copy(ram_matrices.M_ind) : M_indices = nothing #preallocate arrays - A_pre = zeros(n_nod, n_nod) - S_pre = zeros(n_nod, n_nod) - !isnothing(M_indices) ? M_pre = zeros(n_nod) : M_pre = nothing + A_pre = zeros(n_var, n_var) + S_pre = zeros(n_var, n_var) + M_pre = !isnothing(M_indices) ? zeros(n_var) : nothing set_RAMConstants!(A_pre, S_pre, M_pre, ram_matrices.constants) A_pre = check_acyclic(A_pre, n_par, A_indices) # pre-allocate some matrices - Σ = zeros(n_var, n_var) - F⨉I_A⁻¹ = zeros(n_var, n_nod) - F⨉I_A⁻¹S = zeros(n_var, n_nod) + Σ = zeros(n_obs, n_obs) + F⨉I_A⁻¹ = zeros(n_obs, n_var) + F⨉I_A⁻¹S = zeros(n_obs, n_var) I_A = similar(A_pre) if gradient - ∇A = get_matrix_derivative(A_indices, parameters, n_nod^2) - ∇S = get_matrix_derivative(S_indices, parameters, n_nod^2) + ∇A = get_matrix_derivative(A_indices, parameters, n_var^2) + ∇S = get_matrix_derivative(S_indices, parameters, n_var^2) else ∇A = nothing ∇S = nothing @@ -150,12 +149,12 @@ function RAM(; has_meanstructure = Val(true) if gradient - ∇M = get_matrix_derivative(M_indices, parameters, n_nod) + ∇M = get_matrix_derivative(M_indices, parameters, n_var) else ∇M = nothing end - μ = zeros(n_var) + μ = zeros(n_obs) else has_meanstructure = Val(false) @@ -173,7 +172,6 @@ function RAM(; μ, M_pre, - n_par, ram_matrices, has_meanstructure, @@ -285,7 +283,7 @@ objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_means ############################################################################################ identifier(imply::RAM) = imply.identifier -n_par(imply::RAM) = imply.n_par +n_par(imply::RAM) = nparams(imply.ram_matrices) function update_observed(imply::RAM, observed::SemObserved; kwargs...) if n_man(observed) == size(imply.Σ, 1) From 86b7a7f7ce6bc476ba2b121e748c6cc896947aeb Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 23:55:42 -0700 Subject: [PATCH 043/174] RAMSymbolic: use vars API --- src/imply/RAM/symbolic.jl | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 2c497bb1e..62d382fd5 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -62,7 +62,7 @@ and for models with a meanstructure, the model implied means are computed as \mu = F(I-A)^{-1}M ``` """ -struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V, V2, F4, A4, F5, A5, D1, B} <: SemImplySymbolic +struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, D1, B} <: SemImplySymbolic Σ_function::F1 ∇Σ_function::F2 ∇²Σ_function::F3 @@ -72,7 +72,6 @@ struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V, V2, F4, A4, F5, A5, D1 Σ_symbolic::S1 ∇Σ_symbolic::S2 ∇²Σ_symbolic::S3 - n_par::V ram_matrices::V2 μ_function::F4 μ::A4 @@ -99,15 +98,16 @@ function RAMSymbolic(; ram_matrices = convert(RAMMatrices, specification) identifier = StructuralEquationModels.identifier(ram_matrices) - n_par = length(ram_matrices.parameters) - n_var, n_nod = ram_matrices.size_F + n_par = nparams(ram_matrices) + n_obs = nobserved_vars(ram_matrices) + n_var = nvars(ram_matrices) par = (Symbolics.@variables θ[1:n_par])[1] - A = zeros(Num, n_nod, n_nod) - S = zeros(Num, n_nod, n_nod) - !isnothing(ram_matrices.M_ind) ? M = zeros(Num, n_nod) : M = nothing - F = zeros(ram_matrices.size_F); F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 + A = zeros(Num, n_var, n_var) + S = zeros(Num, n_var, n_var) + !isnothing(ram_matrices.M_ind) ? M = zeros(Num, n_var) : M = nothing + F = zeros(ram_matrices.size_F); F[CartesianIndex.(1:n_obs, ram_matrices.F_ind)] .= 1.0 set_RAMConstants!(A, S, M, ram_matrices.constants) fill_A_S_M!(A, S, M, ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.M_ind, par) @@ -140,7 +140,6 @@ function RAMSymbolic(; if hessian & !approximate_hessian n_sig = length(Σ_symbolic) - n_par = size(par, 1) ∇²Σ_symbolic_vec = [Symbolics.sparsehessian(σᵢ, [par...]) for σᵢ in vec(Σ_symbolic)] @variables J[1:n_sig] @@ -189,7 +188,6 @@ function RAMSymbolic(; Σ_symbolic, ∇Σ_symbolic, ∇²Σ_symbolic, - n_par, ram_matrices, μ_function, μ, @@ -235,7 +233,7 @@ objective_gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, p ############################################################################################ identifier(imply::RAMSymbolic) = imply.identifier -n_par(imply::RAMSymbolic) = imply.n_par +n_par(imply::RAMSymbolic) = nparams(imply.ram_matrices) function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) if Int(n_man(observed)) == size(imply.Σ, 1) From 4108f7ab5ba9b0088158e0da485f9f3dc4b97ede Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 23:55:42 -0700 Subject: [PATCH 044/174] start_simple(): use vars API --- src/additional_functions/start_val/start_simple.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 248b517df..6c9fb2018 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -63,18 +63,18 @@ function start_simple( start_means = 0.0, kwargs...) - A_ind, S_ind, F_ind, M_ind, parameters = + A_ind, S_ind, F_ind, M_ind, n_par = ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.F_ind, ram_matrices.M_ind, - ram_matrices.parameters + nparams(ram_matrices) - n_par = length(parameters) start_val = zeros(n_par) - n_var, n_nod = ram_matrices.size_F + n_obs = nobserved_vars(ram_matrices) + n_var = nvars(ram_matrices) - C_indices = CartesianIndices((n_nod, n_nod)) + C_indices = CartesianIndices((n_var, n_var)) for i in 1:n_par if length(S_ind[i]) != 0 From 1403932b132f630263f9ef857564667a12305a47 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 23:55:42 -0700 Subject: [PATCH 045/174] starts_fabin3: use vars API --- .../start_val/start_fabin3.jl | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index fdc32a6bd..5124a682d 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -51,19 +51,19 @@ end function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) - A_ind, S_ind, F_ind, M_ind, parameters = - ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.F_ind, - ram_matrices.M_ind, - ram_matrices.parameters - - n_par = length(parameters) + A_ind, S_ind, F_ind, M_ind, n_par = + ram_matrices.A_ind, + ram_matrices.S_ind, + ram_matrices.F_ind, + ram_matrices.M_ind, + nparams(ram_matrices) + start_val = zeros(n_par) - n_var, n_nod = ram_matrices.size_F - n_latent = n_nod - n_var + n_obs = nobserved_vars(ram_matrices) + n_var = nvars(ram_matrices) + n_latent = nlatent_vars(ram_matrices) - C_indices = CartesianIndices((n_nod, n_nod)) + C_indices = CartesianIndices((n_var, n_var)) # check in which matrix each parameter appears @@ -71,7 +71,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) #= in_S = length.(S_ind) .!= 0 in_A = length.(A_ind) .!= 0 - A_ind_c = [linear2cartesian(ind, (n_nod, n_nod)) for ind in A_ind] + A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] in_Λ = [any(ind[2] .∈ F_ind) for ind in A_ind_c] if !isnothing(M) @@ -98,7 +98,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) # set loadings constants = ram_matrices.constants - A_ind_c = [linear2cartesian(ind, (n_nod, n_nod)) for ind in A_ind] + A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c]) function calculate_lambda(ref::Integer, indicator::Integer, @@ -117,7 +117,7 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end end - for i ∈ setdiff(1:n_nod, F_ind) + for i ∈ setdiff(1:n_var, F_ind) reference = Int64[] indicators = Int64[] indicator2parampos = Dict{Int, Int}() From 085b0dfc891d657d9d68c471232c749a802e5924 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 11 Mar 2024 22:03:40 -0700 Subject: [PATCH 046/174] remove get_colnames() replaced by observed_vars() --- src/StructuralEquationModels.jl | 1 - src/observed/covariance.jl | 4 +++- src/observed/data.jl | 4 +++- src/observed/get_colnames.jl | 22 ---------------------- src/observed/missing.jl | 4 +++- 5 files changed, 9 insertions(+), 26 deletions(-) delete mode 100644 src/observed/get_colnames.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index c7b2750f5..15ce40eb1 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -24,7 +24,6 @@ include("frontend/fit/summary.jl") # pretty printing include("frontend/pretty_printing.jl") # observed -include("observed/get_colnames.jl") include("observed/covariance.jl") include("observed/data.jl") include("observed/missing.jl") diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index fa73f5a3e..9f28d112e 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -71,7 +71,9 @@ function SemObservedCovariance(; end - if isnothing(spec_colnames) spec_colnames = get_colnames(specification) end + if isnothing(spec_colnames) && !isnothing(specification) + spec_colnames = observed_vars(specification) + end if !isnothing(spec_colnames) & isnothing(obs_colnames) diff --git a/src/observed/data.jl b/src/observed/data.jl index 49aa1fee4..565498497 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -69,7 +69,9 @@ function SemObservedData(; kwargs...) - if isnothing(spec_colnames) spec_colnames = get_colnames(specification) end + if isnothing(spec_colnames) && !isnothing(specification) + spec_colnames = observed_vars(specification) + end if !isnothing(spec_colnames) if isnothing(obs_colnames) diff --git a/src/observed/get_colnames.jl b/src/observed/get_colnames.jl deleted file mode 100644 index 599588ce8..000000000 --- a/src/observed/get_colnames.jl +++ /dev/null @@ -1,22 +0,0 @@ -# specification colnames (only observed) -function get_colnames(specification::ParameterTable) - colnames = isempty(specification.variables.sorted) ? - specification.variables.observed : - filter(in(Set(specification.variables.observed)), - specification.variables.sorted) - return colnames -end - -function get_colnames(specification::RAMMatrices) - if isnothing(specification.colnames) - @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" - return nothing - else - colnames = specification.colnames[specification.F_ind] - return colnames - end -end - -function get_colnames(specification::Nothing) - return nothing -end \ No newline at end of file diff --git a/src/observed/missing.jl b/src/observed/missing.jl index be4b49784..0a7d09da6 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -94,7 +94,9 @@ function SemObservedMissing(; kwargs...) - if isnothing(spec_colnames) spec_colnames = get_colnames(specification) end + if isnothing(spec_colnames) && !isnothing(specification) + spec_colnames = observed_vars(specification) + end if !isnothing(spec_colnames) if isnothing(obs_colnames) From d66452d6539fa53d1f6e2e504ef114fde116d0ab Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:40:33 -0700 Subject: [PATCH 047/174] remove get_n_nodes() replaced by nvars() --- src/loss/ML/FIML.jl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 71279d301..9988bdba1 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -64,7 +64,7 @@ function SemFIML(;observed, specification, kwargs...) ∇ind = vec(CartesianIndices(Array{Float64}(undef, nman, nman))) ∇ind = [findall(x -> !(x[1] ∈ ind || x[2] ∈ ind), ∇ind) for ind in patterns_not(observed)] - commutation_indices = get_commutation_lookup(get_n_nodes(specification)^2) + commutation_indices = get_commutation_lookup(nvars(specification)^2) return SemFIML( inverses, @@ -254,8 +254,3 @@ function check_fiml(semfiml, model) a = cholesky!(Symmetric(semfiml.imp_inv); check = false) return isposdef(a) end - -get_n_nodes(specification::RAMMatrices) = specification.size_F[2] -get_n_nodes(specification::ParameterTable) = - length(specification.variables[:observed_vars]) + - length(specification.variables[:latent_vars]) From b44a9e827795e90f3eff3569014406c95d269585 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:02:00 -0700 Subject: [PATCH 048/174] n_par() -> nparams() for clarity and aligning to Julia naming conventions --- src/StructuralEquationModels.jl | 4 ++-- src/additional_functions/simulation.jl | 2 +- src/frontend/fit/fitmeasures/AIC.jl | 2 +- src/frontend/fit/fitmeasures/BIC.jl | 2 +- src/frontend/fit/fitmeasures/df.jl | 2 +- src/frontend/fit/fitmeasures/fit_measures.jl | 2 +- src/frontend/fit/fitmeasures/n_par.jl | 18 +++++++++--------- src/frontend/fit/summary.jl | 2 +- src/frontend/specification/Sem.jl | 2 +- src/imply/RAM/generic.jl | 4 ++-- src/imply/RAM/symbolic.jl | 4 ++-- src/imply/empty.jl | 4 ++-- src/loss/regularization/ridge.jl | 12 ++++++------ src/types.jl | 1 - test/examples/helper.jl | 4 ++-- test/examples/political_democracy/by_parts.jl | 2 +- .../recover_parameters_twofact.jl | 2 +- 17 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 15ce40eb1..0f6f0581b 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -105,9 +105,9 @@ export AbstractSem, get_identifier_indices, RAMMatrices, RAMMatrices!, - identifier, + identifier, nparams, fit_measures, - AIC, BIC, χ², df, fit_measures, minus2ll, n_par, n_obs, p_value, RMSEA, n_man, + AIC, BIC, χ², df, fit_measures, minus2ll, n_obs, p_value, RMSEA, n_man, EmMVNModel, se_hessian, se_bootstrap, example_data, diff --git a/src/additional_functions/simulation.jl b/src/additional_functions/simulation.jl index 30ab29b76..313a74884 100644 --- a/src/additional_functions/simulation.jl +++ b/src/additional_functions/simulation.jl @@ -65,7 +65,7 @@ function swap_observed( # update imply imply = update_observed(imply, new_observed; kwargs...) kwargs[:imply] = imply - kwargs[:n_par] = n_par(imply) + kwargs[:nparams] = nparams(imply) # update loss loss = update_observed(loss, new_observed; kwargs...) diff --git a/src/frontend/fit/fitmeasures/AIC.jl b/src/frontend/fit/fitmeasures/AIC.jl index 7e9dc9bbe..d08a7912e 100644 --- a/src/frontend/fit/fitmeasures/AIC.jl +++ b/src/frontend/fit/fitmeasures/AIC.jl @@ -3,4 +3,4 @@ Return the akaike information criterion. """ -AIC(sem_fit::SemFit) = minus2ll(sem_fit) + 2n_par(sem_fit) \ No newline at end of file +AIC(sem_fit::SemFit) = minus2ll(sem_fit) + 2nparams(sem_fit) \ No newline at end of file diff --git a/src/frontend/fit/fitmeasures/BIC.jl b/src/frontend/fit/fitmeasures/BIC.jl index 3bb2e16ee..ecb40798a 100644 --- a/src/frontend/fit/fitmeasures/BIC.jl +++ b/src/frontend/fit/fitmeasures/BIC.jl @@ -3,4 +3,4 @@ Return the bayesian information criterion. """ -BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(n_obs(sem_fit))*n_par(sem_fit) \ No newline at end of file +BIC(sem_fit::SemFit) = minus2ll(sem_fit) + log(n_obs(sem_fit))*nparams(sem_fit) \ No newline at end of file diff --git a/src/frontend/fit/fitmeasures/df.jl b/src/frontend/fit/fitmeasures/df.jl index 2680ec6bf..501bc3d26 100644 --- a/src/frontend/fit/fitmeasures/df.jl +++ b/src/frontend/fit/fitmeasures/df.jl @@ -8,7 +8,7 @@ function df end df(sem_fit::SemFit) = df(sem_fit.model) -df(model::AbstractSem) = n_dp(model) - n_par(model) +df(model::AbstractSem) = n_dp(model) - nparams(model) function n_dp(model::AbstractSemSingle) nman = n_man(model) diff --git a/src/frontend/fit/fitmeasures/fit_measures.jl b/src/frontend/fit/fitmeasures/fit_measures.jl index f915a2ed3..8918ee446 100644 --- a/src/frontend/fit/fitmeasures/fit_measures.jl +++ b/src/frontend/fit/fitmeasures/fit_measures.jl @@ -1,7 +1,7 @@ fit_measures(sem_fit) = fit_measures( sem_fit, - n_par, + nparams, df, AIC, BIC, diff --git a/src/frontend/fit/fitmeasures/n_par.jl b/src/frontend/fit/fitmeasures/n_par.jl index 08d2da5ba..c569892a3 100644 --- a/src/frontend/fit/fitmeasures/n_par.jl +++ b/src/frontend/fit/fitmeasures/n_par.jl @@ -2,19 +2,19 @@ ### get number of parameters ############################################################################################ """ - n_par(sem_fit::SemFit) - n_par(model::AbstractSemSingle) - n_par(model::SemEnsemble) - n_par(identifier::Dict) + nparams(sem_fit::SemFit) + nparams(model::AbstractSemSingle) + nparams(model::SemEnsemble) + nparams(identifier::Dict) Return the number of parameters. """ -function n_par end +function nparams end -n_par(fit::SemFit) = n_par(fit.model) +nparams(fit::SemFit) = nparams(fit.model) -n_par(model::AbstractSemSingle) = n_par(model.imply) +nparams(model::AbstractSemSingle) = nparams(model.imply) -n_par(model::SemEnsemble) = n_par(model.identifier) +nparams(model::SemEnsemble) = nparams(model.identifier) -n_par(identifier::Dict) = length(identifier) \ No newline at end of file +nparams(identifier::Dict) = length(identifier) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index 33d70113f..f3db9912b 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -8,7 +8,7 @@ function sem_summary(sem_fit::SemFit; show_fitmeasures = false, color = :light_c println("Convergence: $(convergence(sem_fit))") println("No. iterations/evaluations: $(n_iterations(sem_fit))") print("\n") - println("Number of parameters: $(n_par(sem_fit))") + println("Number of parameters: $(nparams(sem_fit))") println("Number of observations: $(n_obs(sem_fit))") print("\n") printstyled("----------------------------------- Model ----------------------------------- \n"; color = color) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 04fadfddb..96f07f671 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -69,7 +69,7 @@ function get_fields!(kwargs, observed, imply, loss, optimizer) end kwargs[:imply] = imply - kwargs[:n_par] = n_par(imply) + kwargs[:nparams] = nparams(imply) # loss loss = get_SemLoss(loss; kwargs...) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index dbe8637e2..181b47d57 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -35,7 +35,7 @@ and for models with a meanstructure, the model implied means are computed as ## Interfaces - `identifier(::RAM) `-> Dict containing the parameter labels and their position -- `n_par(::RAM)` -> Number of parameters +- `nparams(::RAM)` -> Number of parameters - `Σ(::RAM)` -> model implied covariance matrix - `μ(::RAM)` -> model implied mean vector @@ -283,7 +283,7 @@ objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_means ############################################################################################ identifier(imply::RAM) = imply.identifier -n_par(imply::RAM) = nparams(imply.ram_matrices) +nparams(imply::RAM) = nparams(imply.ram_matrices) function update_observed(imply::RAM, observed::SemObserved; kwargs...) if n_man(observed) == size(imply.Σ, 1) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 62d382fd5..5f0ca0120 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -30,7 +30,7 @@ Subtype of `SemImply`. ## Interfaces - `identifier(::RAMSymbolic) `-> Dict containing the parameter labels and their position -- `n_par(::RAMSymbolic)` -> Number of parameters +- `nparams(::RAMSymbolic)` -> number of parameters - `Σ(::RAMSymbolic)` -> model implied covariance matrix - `μ(::RAMSymbolic)` -> model implied mean vector @@ -233,7 +233,7 @@ objective_gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, p ############################################################################################ identifier(imply::RAMSymbolic) = imply.identifier -n_par(imply::RAMSymbolic) = nparams(imply.ram_matrices) +nparams(imply::RAMSymbolic) = nparams(imply.ram_matrices) function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) if Int(n_man(observed)) == size(imply.Σ, 1) diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 65d0e3259..31c550062 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -20,7 +20,7 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the ## Interfaces - `identifier(::RAMSymbolic) `-> Dict containing the parameter labels and their position -- `n_par(::RAMSymbolic)` -> Number of parameters +- `nparams(::RAMSymbolic)` -> Number of parameters ## Implementation Subtype of `SemImply`. @@ -59,6 +59,6 @@ hessian!(imply::ImplyEmpty, par, model) = nothing ############################################################################################ identifier(imply::ImplyEmpty) = imply.identifier -n_par(imply::ImplyEmpty) = imply.n_par +nparams(imply::ImplyEmpty) = imply.nparams update_observed(imply::ImplyEmpty, observed::SemObserved; kwargs...) = imply \ No newline at end of file diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 09d1d3933..f5f29cfb9 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -8,18 +8,18 @@ Ridge regularization. # Constructor - SemRidge(;α_ridge, which_ridge, n_par, parameter_type = Float64, imply = nothing, kwargs...) + SemRidge(;α_ridge, which_ridge, nparams, parameter_type = Float64, imply = nothing, kwargs...) # Arguments - `α_ridge`: hyperparameter for penalty term - `which_ridge::Vector`: Vector of parameter labels (Symbols) or indices that indicate which parameters should be regularized. -- `n_par::Int`: number of parameters of the model +- `nparams::Int`: number of parameters of the model - `imply::SemImply`: imply part of the model - `parameter_type`: type of the parameters # Examples ```julia -my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], n_par = 30, imply = my_imply) +my_ridge = SemRidge(;α_ridge = 0.02, which_ridge = [:λ₁, :λ₂, :ω₂₃], nparams = 30, imply = my_imply) ``` # Interfaces @@ -45,7 +45,7 @@ end function SemRidge(; α_ridge, which_ridge, - n_par, + nparams, parameter_type = Float64, imply = nothing, kwargs...) @@ -64,8 +64,8 @@ function SemRidge(; which, which_H, - zeros(parameter_type, n_par), - zeros(parameter_type, n_par, n_par)) + zeros(parameter_type, nparams), + zeros(parameter_type, nparams, nparams)) end ############################################################################################ diff --git a/src/types.jl b/src/types.jl index a75b5b45a..e4e1ab2f1 100644 --- a/src/types.jl +++ b/src/types.jl @@ -167,7 +167,6 @@ end function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing, kwargs...) n = length(models) - npar = n_par(models[1]) # default weights diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 9cdf18bf7..6d4dc9d18 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -47,7 +47,7 @@ fitmeasure_names_ml = Dict( :df => "df", :χ² => "chisq", :p_value => "pvalue", - :n_par => "npar", + :nparams => "npar", :RMSEA => "rmsea", ) @@ -55,7 +55,7 @@ fitmeasure_names_ls = Dict( :df => "df", :χ² => "chisq", :p_value => "pvalue", - :n_par => "npar", + :nparams => "npar", :RMSEA => "rmsea", ) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index daddffe94..22caf33c9 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -15,7 +15,7 @@ ml = SemML(observed = observed) wls = SemWLS(observed = observed) -ridge = SemRidge(α_ridge = .001, which_ridge = 16:20, n_par = 31) +ridge = SemRidge(α_ridge = .001, which_ridge = 16:20, nparams = 31) constant = SemConstant(constant_loss = 3.465) diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 8bbcad725..f0b099f08 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -51,7 +51,7 @@ Random.seed!(1234) x = transpose(rand(true_dist, 100000)) semobserved = SemObservedData(data = x, specification = nothing) -loss_ml = SemLoss(SEM.SemML(;observed = semobserved, n_par = length(start))) +loss_ml = SemLoss(SEM.SemML(;observed = semobserved, nparams = length(start))) optimizer = SemOptimizerOptim( From 9b58bbff54eab9b971ca5792f6f5311fb1128467 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 17:28:21 -0700 Subject: [PATCH 049/174] MeanStructure, HessianEvaluation traits * replace has_meanstrcture and approximate_hessian fields with trait-like typeparams * remove methods for has_meanstructure-based dispatch --- src/StructuralEquationModels.jl | 2 + src/imply/RAM/generic.jl | 50 ++--- src/imply/RAM/symbolic.jl | 35 ++- src/imply/empty.jl | 2 +- src/loss/ML/FIML.jl | 2 +- src/loss/ML/ML.jl | 262 ++++++++++------------- src/loss/WLS/WLS.jl | 137 ++++++------ src/loss/constant/constant.jl | 2 +- src/loss/regularization/ridge.jl | 2 +- src/types.jl | 31 ++- test/examples/multigroup/build_models.jl | 2 +- 11 files changed, 246 insertions(+), 281 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 0f6f0581b..0c361c835 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -81,6 +81,8 @@ include("frontend/fit/standard_errors/bootstrap.jl") export AbstractSem, AbstractSemSingle, AbstractSemCollection, Sem, SemFiniteDiff, SemEnsemble, + MeanStructure, NoMeanStructure, HasMeanStructure, + HessianEvaluation, ExactHessian, ApproximateHessian, SemImply, RAMSymbolic, RAMSymbolicZ, RAM, ImplyEmpty, imply, start_val, diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 181b47d57..58afd3191 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -65,7 +65,7 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3, B, D} <: SemImply +mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3, D} <: SemImply{MS, ExactHessian} Σ::A1 A::A2 S::A3 @@ -74,7 +74,6 @@ mutable struct RAM{A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S M::A6 ram_matrices::V2 - has_meanstructure::B A_indices::I1 S_indices::I2 @@ -92,12 +91,13 @@ mutable struct RAM{A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S identifier::D end -using StructuralEquationModels - ############################################################################################ ### Constructors ############################################################################################ + +RAM{MS}(args...) where MS <: MeanStructure = RAM{MS, map(typeof, args)...}(args...) + function RAM(; specification::SemSpecification, #vech = false, @@ -145,8 +145,7 @@ function RAM(; # μ if meanstructure - - has_meanstructure = Val(true) + MS = HasMeanStructure if gradient ∇M = get_matrix_derivative(M_indices, parameters, n_var) @@ -157,14 +156,14 @@ function RAM(; μ = zeros(n_obs) else - has_meanstructure = Val(false) + MS = NoMeanStructure M_indices = nothing M_pre = nothing μ = nothing ∇M = nothing end - return RAM( + return RAM{MS}( Σ, A_pre, S_pre, @@ -173,7 +172,6 @@ function RAM(; M_pre, ram_matrices, - has_meanstructure, A_indices, S_indices, @@ -196,14 +194,8 @@ end ### methods ############################################################################################ -# dispatch on meanstructure -objective!(imply::RAM, par, model::AbstractSemSingle) = - objective!(imply, par, model, imply.has_meanstructure) -gradient!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model, imply.has_meanstructure) - # objective and gradient -function objective!(imply::RAM, parameters, model, has_meanstructure::Val{T}) where T +function objective!(imply::RAM, parameters, model) fill_A_S_M!( imply.A, @@ -229,13 +221,13 @@ function objective!(imply::RAM, parameters, model, has_meanstructure::Val{T}) wh imply.S, imply.F⨉I_A⁻¹S) - if T + if MeanStructure(imply) === HasMeanStructure μ_RAM!(imply.μ, imply.F⨉I_A⁻¹, imply.M) end end -function gradient!(imply::RAM, parameters, model::AbstractSemSingle, has_meanstructure::Val{T}) where T +function gradient!(imply::RAM, parameters, model::AbstractSemSingle) fill_A_S_M!( imply.A, @@ -261,22 +253,22 @@ function gradient!(imply::RAM, parameters, model::AbstractSemSingle, has_meanstr imply.S, imply.F⨉I_A⁻¹S) - if T + if MeanStructure(imply) === HasMeanStructure μ_RAM!(imply.μ, imply.F⨉I_A⁻¹, imply.M) end end -hessian!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) -objective_gradient!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) -objective_hessian!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) -gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) -objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle, has_meanstructure) = - gradient!(imply, par, model, has_meanstructure) +hessian!(imply::RAM, par, model::AbstractSemSingle) = + gradient!(imply, par, model) +objective_gradient!(imply::RAM, par, model::AbstractSemSingle) = + gradient!(imply, par, model) +objective_hessian!(imply::RAM, par, model::AbstractSemSingle) = + gradient!(imply, par, model) +gradient_hessian!(imply::RAM, par, model::AbstractSemSingle) = + gradient!(imply, par, model) +objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle) = + gradient!(imply, par, model) ############################################################################################ ### Recommended methods diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 5f0ca0120..da1e7e595 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -62,7 +62,7 @@ and for models with a meanstructure, the model implied means are computed as \mu = F(I-A)^{-1}M ``` """ -struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, D1, B} <: SemImplySymbolic +struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, D1} <: SemImplySymbolic{MS,ExactHessian} Σ_function::F1 ∇Σ_function::F2 ∇²Σ_function::F3 @@ -78,13 +78,14 @@ struct RAMSymbolic{F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, D1, B ∇μ_function::F5 ∇μ::A5 identifier::D1 - has_meanstructure::B end ############################################################################################ ### Constructors ############################################################################################ +RAMSymbolic{MS}(args...) where MS <: MeanStructure = RAMSymbolic{MS, map(typeof, args)...}(args...) + function RAMSymbolic(; specification::SemSpecification, loss_types = nothing, @@ -96,7 +97,6 @@ function RAMSymbolic(; kwargs...) ram_matrices = convert(RAMMatrices, specification) - identifier = StructuralEquationModels.identifier(ram_matrices) n_par = nparams(ram_matrices) n_obs = nobserved_vars(ram_matrices) @@ -138,7 +138,7 @@ function RAMSymbolic(; ∇Σ = nothing end - if hessian & !approximate_hessian + if hessian && !approximate_hessian n_sig = length(Σ_symbolic) ∇²Σ_symbolic_vec = [Symbolics.sparsehessian(σᵢ, [par...]) for σᵢ in vec(Σ_symbolic)] @@ -158,7 +158,7 @@ function RAMSymbolic(; # μ if meanstructure - has_meanstructure = Val(true) + MS = HasMeanStructure μ_symbolic = get_μ_symbolic_RAM(M, A, F) μ_function = Symbolics.build_function(μ_symbolic, par, expression=Val{false})[2] μ = zeros(size(μ_symbolic)) @@ -171,14 +171,14 @@ function RAMSymbolic(; ∇μ = nothing end else - has_meanstructure = Val(false) + MS = NoMeanStructure μ_function = nothing μ = nothing ∇μ_function = nothing ∇μ = nothing end - return RAMSymbolic( + return RAMSymbolic{MS}( Σ_function, ∇Σ_function, ∇²Σ_function, @@ -193,8 +193,7 @@ function RAMSymbolic(; μ, ∇μ_function, ∇μ, - identifier, - has_meanstructure + identifier(ram_matrices), ) end @@ -202,23 +201,21 @@ end ### objective, gradient, hessian ############################################################################################ -# dispatch on meanstructure -objective!(imply::RAMSymbolic, par, model) = - objective!(imply, par, model, imply.has_meanstructure) -gradient!(imply::RAMSymbolic, par, model) = - gradient!(imply, par, model, imply.has_meanstructure) - # objective -function objective!(imply::RAMSymbolic, par, model, has_meanstructure::Val{T}) where T +function objective!(imply::RAMSymbolic, par, model) imply.Σ_function(imply.Σ, par) - T && imply.μ_function(imply.μ, par) + if MeanStructure(imply) === HasMeanStructure + imply.μ_function(imply.μ, par) + end end # gradient -function gradient!(imply::RAMSymbolic, par, model, has_meanstructure::Val{T}) where T +function gradient!(imply::RAMSymbolic, par, model) objective!(imply, par, model, imply.has_meanstructure) imply.∇Σ_function(imply.∇Σ, par) - T && imply.∇μ_function(imply.∇μ, par) + if MeanStructure(imply) === HasMeanStructure + imply.∇μ_function(imply.∇μ, par) + end end # other methods diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 31c550062..9a0231536 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -25,7 +25,7 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the ## Implementation Subtype of `SemImply`. """ -struct ImplyEmpty{V, V2} <: SemImply +struct ImplyEmpty{V, V2} <: SemImply{NoMeanStructure,ExactHessian} identifier::V2 n_par::V end diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 9988bdba1..c49b8f916 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -24,7 +24,7 @@ Analytic gradients are available. ## Implementation Subtype of `SemLossFunction`. """ -mutable struct SemFIML{INV, C, L, O, M, IM, I, T, U, W} <: SemLossFunction +mutable struct SemFIML{INV, C, L, O, M, IM, I, T, U, W} <: SemLossFunction{ExactHessian} inverses::INV #preallocated inverses of imp_cov choleskys::C #preallocated choleskys logdets::L #logdets of implied covmats diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 4a47f7847..0a9169bae 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -27,62 +27,61 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemML{INV,M,M2,B, V} <: SemLossFunction - Σ⁻¹::INV +struct SemML{HE<:HessianEvaluation,INV,M,M2} <: SemLossFunction{HE} + Σ⁻¹::INV Σ⁻¹Σₒ::M meandiff::M2 - approximate_hessian::B - has_meanstructure::V end ############################################################################################ ### Constructors ############################################################################################ -function SemML(;observed, meanstructure = false, approximate_hessian = false, kwargs...) - isnothing(obs_mean(observed)) ? - meandiff = nothing : - meandiff = copy(obs_mean(observed)) - return SemML( - similar(obs_cov(observed)), - similar(obs_cov(observed)), - meandiff, - approximate_hessian, - Val(meanstructure) - ) +SemML{HE}(args...) where {HE <: HessianEvaluation} = + SemML{HE, map(typeof, args)...}(args...) + +function SemML(; observed::SemObserved, + approximate_hessian::Bool = false, + kwargs...) + obsmean = obs_mean(observed) + obscov = obs_cov(observed) + meandiff = isnothing(obsmean) ? nothing : copy(obsmean) + + return SemML{approximate_hessian ? ApproximateHessian : ExactHessian}( + similar(obscov), similar(obscov), + meandiff) end ############################################################################################ ### objective, gradient, hessian methods ############################################################################################ -# first, dispatch for meanstructure -objective!(semml::SemML, par, model::AbstractSemSingle) = - objective!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) -gradient!(semml::SemML, par, model::AbstractSemSingle) = - gradient!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) -hessian!(semml::SemML, par, model::AbstractSemSingle) = - hessian!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) -objective_gradient!(semml::SemML, par, model::AbstractSemSingle) = - objective_gradient!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) -objective_hessian!(semml::SemML, par, model::AbstractSemSingle) = - objective_hessian!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) -gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = - gradient_hessian!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) -objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = - objective_gradient_hessian!(semml::SemML, par, model, semml.has_meanstructure, imply(model)) +# dispatch for SemImply +objective!(semml::SemML, par, model::AbstractSemSingle) = + objective!(semml, par, model, imply(model)) +gradient!(semml::SemML, par, model::AbstractSemSingle) = + gradient!(semml, par, model, imply(model)) +hessian!(semml::SemML, par, model::AbstractSemSingle) = + hessian!(semml, par, model, imply(model)) +objective_gradient!(semml::SemML, par, model::AbstractSemSingle) = + objective_gradient!(semml, par, model, imply(model)) +objective_hessian!(semml::SemML, par, model::AbstractSemSingle) = + objective_hessian!(semml, par, model, imply(model)) +gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = + gradient_hessian!(semml, par, model, imply(model)) +objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = + objective_gradient_hessian!(semml, par, model, imply(model)) ############################################################################################ ### Symbolic Imply Types function objective!( semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::SemImplySymbolic) where T - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), + par, + model::AbstractSemSingle, + imp::SemImplySymbolic) + + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), μ = μ(imply(model)), μₒ = obs_mean(observed(model)) copyto!(Σ⁻¹, Σ) @@ -92,7 +91,7 @@ function objective!( Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) #mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ return ld + dot(Σ⁻¹, Σₒ) + dot(μ₋, Σ⁻¹, μ₋) else @@ -102,42 +101,42 @@ function objective!( end function gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::SemImplySymbolic) where T + semml::SemML, + par, + model::AbstractSemSingle, + imp::SemImplySymbolic) - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), ∇Σ = ∇Σ(imply(model)), μ = μ(imply(model)), ∇μ = ∇μ(imply(model)), μₒ = obs_mean(observed(model)) - + copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) isposdef(Σ_chol) || return ones(eltype(par), size(par)) Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ gradient = vec(Σ⁻¹*(I - Σₒ*Σ⁻¹ - μ₋*μ₋ᵀΣ⁻¹))'*∇Σ - 2*μ₋ᵀΣ⁻¹*∇μ - return gradient' else gradient = (vec(Σ⁻¹)-vec(Σ⁻¹Σₒ*Σ⁻¹))'*∇Σ - return gradient' end + return gradient' end end function hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{false}, + semml::SemML, + par, + model::AbstractSemSingle, imp::SemImplySymbolic) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end - let Σ = Σ(imply(model)), ∇Σ = ∇Σ(imply(model)), Σₒ = obs_cov(observed(model)), + let Σ = Σ(imply(model)), ∇Σ = ∇Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) @@ -147,7 +146,7 @@ function hessian!( return diagm(fill(one(eltype(par)), length(par))) Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - if semml.approximate_hessian + if HessianEvaluation(semml) === ApproximateHessian hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ else mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) @@ -160,29 +159,19 @@ function hessian!( hessian = ∇Σ'*H_outer*∇Σ hessian .+= ∇²Σ end - + return hessian end end -function hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, +function objective_gradient!( + semml::SemML, + par, + model::AbstractSemSingle, imp::SemImplySymbolic) - throw(DomainError(H, "hessian of ML + meanstructure is not available")) -end -function objective_gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::SemImplySymbolic) where T - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), μ = μ(imply(model)), μₒ = obs_mean(observed(model)), + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), + Σ⁻¹ = Σ⁻¹(semml), μ = μ(imply(model)), μₒ = obs_mean(observed(model)), ∇Σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)) copyto!(Σ⁻¹, Σ) @@ -194,30 +183,30 @@ function objective_gradient!( Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ - + objective = ld + tr(Σ⁻¹Σₒ) + dot(μ₋, Σ⁻¹, μ₋) gradient = vec(Σ⁻¹*(I - Σₒ*Σ⁻¹ - μ₋*μ₋ᵀΣ⁻¹))'*∇Σ - 2*μ₋ᵀΣ⁻¹*∇μ - return objective, gradient' else objective = ld + tr(Σ⁻¹Σₒ) gradient = (vec(Σ⁻¹)-vec(Σ⁻¹Σₒ*Σ⁻¹))'*∇Σ - return objective, gradient' end + return objective, gradient' end end end function objective_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::SemImplySymbolic) where T - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), + semml::SemML, + par, + model::AbstractSemSingle, + imp::SemImplySymbolic) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), ∇Σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)), ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) @@ -231,7 +220,7 @@ function objective_hessian!( mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) objective = ld + tr(Σ⁻¹Σₒ) - if semml.approximate_hessian + if HessianEvaluation(semml) === ApproximateHessian hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ else Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ @@ -249,26 +238,20 @@ function objective_hessian!( end end -function objective_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, - imp::SemImplySymbolic) - throw(DomainError(H, "hessian of ML + meanstructure is not available")) -end - function gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{false}, + semml::SemML, + par, + model::AbstractSemSingle, imp::SemImplySymbolic) - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end + + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), ∇Σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)), ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - + copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) isposdef(Σ_chol) || @@ -281,7 +264,7 @@ function gradient_hessian!( J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' gradient = J*∇Σ - if semml.approximate_hessian + if HessianEvaluation(semml) === ApproximateHessian hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ else # inner @@ -291,31 +274,25 @@ function gradient_hessian!( hessian = ∇Σ'*H_outer*∇Σ hessian .+= ∇²Σ end - + return gradient', hessian end end -function gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, - imp::SemImplySymbolic) - throw(DomainError(H, "hessian of ML + meanstructure is not available")) -end - function objective_gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{false}, + semml::SemML, + par, + model::AbstractSemSingle, imp::SemImplySymbolic) - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of ML + meanstructure is not available")) + end + + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), ∇Σ = ∇Σ(imply(model)), ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - + copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) if !isposdef(Σ_chol) @@ -334,7 +311,7 @@ function objective_gradient_hessian!( J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' gradient = J*∇Σ - if semml.approximate_hessian + if HessianEvaluation(semml) == ApproximateHessian hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ else Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ @@ -345,50 +322,40 @@ function objective_gradient_hessian!( hessian = ∇Σ'*H_outer*∇Σ hessian .+= ∇²Σ end - + return objective, gradient', hessian end end -function objective_gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{true}, - imp::SemImplySymbolic) - throw(DomainError(H, "hessian of ML + meanstructure is not available")) -end - ############################################################################################ ### Non-Symbolic Imply Types # no hessians ------------------------------------------------------------------------------ -function hessian!(semml::SemML, par, model::AbstractSemSingle, has_meanstructure, imp::RAM) +function hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) end -function objective_hessian!(semml::SemML, par, model::AbstractSemSingle, has_meanstructure, imp::RAM) +function objective_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) end -function gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, has_meanstructure, imp::RAM) +function gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) end -function objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, has_meanstructure, imp::RAM) +function objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) end # objective, gradient ---------------------------------------------------------------------- function objective!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::RAM) where T - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), + semml::SemML, + par, + model::AbstractSemSingle, + imp::RAM) + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), μ = μ(imply(model)), μₒ = obs_mean(observed(model)) copyto!(Σ⁻¹, Σ) @@ -398,7 +365,7 @@ function objective!( Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ return ld + tr(Σ⁻¹Σₒ) + dot(μ₋, Σ⁻¹, μ₋) else @@ -407,11 +374,11 @@ function objective!( end end -function gradient!(semml::SemML, par, model::AbstractSemSingle, has_meanstructure::Val{T}, imp::RAM) where T +function gradient!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), S = S(imply(model)), M = M(imply(model)), - F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), I_A⁻¹ = I_A⁻¹(imply(model)), + let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), + Σ⁻¹ = Σ⁻¹(semml), S = S(imply(model)), M = M(imply(model)), + F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), I_A⁻¹ = I_A⁻¹(imply(model)), ∇A = ∇A(imply(model)), ∇S = ∇S(imply(model)), ∇M = ∇M(imply(model)), μ = μ(imply(model)), μₒ = obs_mean(observed(model)) @@ -424,7 +391,7 @@ function gradient!(semml::SemML, par, model::AbstractSemSingle, has_meanstructur C = F⨉I_A⁻¹'*(I-Σₒ*Σ⁻¹)'*Σ⁻¹*F⨉I_A⁻¹ gradient = 2vec(C*S*I_A⁻¹')'∇A + vec(C)'∇S - if T + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ @@ -437,14 +404,13 @@ function gradient!(semml::SemML, par, model::AbstractSemSingle, has_meanstructur end function objective_gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - has_meanstructure::Val{T}, - imp::RAM) where T + semml::SemML, + par, + model::AbstractSemSingle, + imp::RAM) let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), - S = S(imply(model)), M = M(imply(model)), F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), I_A⁻¹ = I_A⁻¹(imply(model)), + S = S(imply(model)), M = M(imply(model)), F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), I_A⁻¹ = I_A⁻¹(imply(model)), ∇A = ∇A(imply(model)), ∇S = ∇S(imply(model)), ∇M = ∇M(imply(model)), μ = μ(imply(model)), μₒ = obs_mean(observed(model)) @@ -463,7 +429,7 @@ function objective_gradient!( C = F⨉I_A⁻¹'*(I-Σₒ*Σ⁻¹)'*Σ⁻¹*F⨉I_A⁻¹ gradient = 2vec(C*S*I_A⁻¹')'∇A + vec(C)'∇S - if T + if MeanStructure(semml) === HasMeanStructure μ₋ = μₒ - μ objective += dot(μ₋, Σ⁻¹, μ₋) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index eae2b5381..03afad85b 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -38,20 +38,21 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemWLS{Vt, St, B, C, B2} <: SemLossFunction +struct SemWLS{HE<:HessianEvaluation,Vt,St,C} <: SemLossFunction{HE} V::Vt σₒ::St - approximate_hessian::B V_μ::C - has_meanstructure::B2 end ############################################################################################ ### Constructors ############################################################################################ -function SemWLS(;observed, wls_weight_matrix = nothing, wls_weight_matrix_mean = nothing, - approximate_hessian = false, meanstructure = false, kwargs...) +SemWLS{HE}(args...) where {HE <: HessianEvaluation} = + SemWLS{HE, map(typeof, args)...}(args...) + +function SemWLS(;observed, wls_weight_matrix = nothing, wls_weight_matrix_mean = nothing, + approximate_hessian = false, meanstructure = false, kwargs...) ind = CartesianIndices(obs_cov(observed)) ind = filter(x -> (x[1] >= x[2]), ind) s = obs_cov(observed)[ind] @@ -71,13 +72,12 @@ function SemWLS(;observed, wls_weight_matrix = nothing, wls_weight_matrix_mean = else wls_weight_matrix_mean = nothing end + HE = approximate_hessian ? ApproximateHessian : AnalyticHessian - return SemWLS( - wls_weight_matrix, - s, - approximate_hessian, + return SemWLS{HE}( + wls_weight_matrix, + s, wls_weight_matrix_mean, - Val(meanstructure) ) end @@ -85,48 +85,30 @@ end ### methods ############################################################################ -objective!(semwls::SemWLS, par, model::AbstractSemSingle) = - objective!(semwls::SemWLS, par, model, semwls.has_meanstructure) -gradient!(semwls::SemWLS, par, model::AbstractSemSingle) = - gradient!(semwls::SemWLS, par, model, semwls.has_meanstructure) -hessian!(semwls::SemWLS, par, model::AbstractSemSingle) = - hessian!(semwls::SemWLS, par, model, semwls.has_meanstructure) - -objective_gradient!(semwls::SemWLS, par, model::AbstractSemSingle) = - objective_gradient!(semwls::SemWLS, par, model, semwls.has_meanstructure) -objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) = - objective_hessian!(semwls::SemWLS, par, model, semwls.has_meanstructure) -gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) = - gradient_hessian!(semwls::SemWLS, par, model, semwls.has_meanstructure) +function objective!(semwls::SemWLS, par, model::AbstractSemSingle) -objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) = - objective_gradient_hessian!(semwls::SemWLS, par, model, semwls.has_meanstructure) + let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, μₒ = obs_mean(observed(model)), V = semwls.V, V_μ = semwls.V_μ, - -function objective!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{T}) where T - - let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, μₒ = obs_mean(observed(model)), V = semwls.V, V_μ = semwls.V_μ, - σ₋ = σₒ - σ - - if T + + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ return dot(σ₋, V, σ₋) + dot(μ₋, V_μ, μ₋) else - return dot(σ₋, V, σ₋) + return dot(σ₋, V, σ₋) end end end -function gradient!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{T}) where T - - let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, +function gradient!(semwls::SemWLS, par, model::AbstractSemSingle) + + let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, μₒ = obs_mean(observed(model)), V = semwls.V, V_μ = semwls.V_μ, ∇σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)) - + σ₋ = σₒ - σ - - if T + + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ return -2*(σ₋'*V*∇σ + μ₋'*V_μ*∇μ)' else @@ -135,19 +117,19 @@ function gradient!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstruct end end -function hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{T}) where T - +function hessian!(semwls::SemWLS, par, model::AbstractSemSingle) + let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, ∇σ = ∇Σ(imply(model)), ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - + σ₋ = σₒ - σ - - if T + + if MeanStructure(imply(model)) === HasMeanStructure throw(DomainError(H, "hessian of WLS with meanstructure is not available")) else hessian = 2*∇σ'*V*∇σ - if !semwls.approximate_hessian + if HessianEvaluation(semwls) === ExactHessian J = -2*(σ₋'*semwls.V)' ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ @@ -157,39 +139,42 @@ function hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructu end end -function objective_gradient!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{T}) where T - - let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, +function objective_gradient!(semwls::SemWLS, par, model::AbstractSemSingle) + + let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, μₒ = obs_mean(observed(model)), V = semwls.V, V_μ = semwls.V_μ, ∇σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)) - + σ₋ = σₒ - σ - - if T + + if MeanStructure(imply(model)) === HasMeanStructure μ₋ = μₒ - μ - objective = dot(σ₋, V, σ₋) + dot(μ₋', V_μ, μ₋) + objective = dot(σ₋, V, σ₋) + dot(μ₋, V_μ, μ₋) gradient = -2*(σ₋'*V*∇σ + μ₋'*V_μ*∇μ)' return objective, gradient else - objective = dot(σ₋, V, σ₋) + objective = dot(σ₋, V, σ₋) gradient = -2*(σ₋'*V*∇σ)' return objective, gradient end end end -function objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{T}) where T - - let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, +function objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of WLS with meanstructure is not available")) + end + + let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, ∇σ = ∇Σ(imply(model)), ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - + σ₋ = σₒ - σ - + objective = dot(σ₋, V, σ₋) hessian = 2*∇σ'*V*∇σ - if !semwls.approximate_hessian + if HessianEvaluation(semwls) === ExactHessian J = -2*(σ₋'*semwls.V)' ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ @@ -199,21 +184,21 @@ function objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_m end end -objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{true}) = - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) +function gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of WLS with meanstructure is not available")) + end -function gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{false}) - let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, ∇σ = ∇Σ(imply(model)), ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - + σ₋ = σₒ - σ - + gradient = -2*(σ₋'*V*∇σ)' hessian = 2*∇σ'*V*∇σ - if !semwls.approximate_hessian + if HessianEvaluation(semwls) === ExactHessian J = -2*(σ₋'*semwls.V)' ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ @@ -223,21 +208,22 @@ function gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_me end end -gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{true}) = - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) +function objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) + + if MeanStructure(imply(model)) === HasMeanStructure + throw(DomainError(H, "hessian of WLS with meanstructure is not available")) + end -function objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{false}) - let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, ∇σ = ∇Σ(imply(model)), ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - + σ₋ = σₒ - σ - - objective = dot(σ₋, V, σ₋) + + objective = dot(σ₋, V, σ₋) gradient = -2*(σ₋'*V*∇σ)' hessian = 2*∇σ'*V*∇σ - if !semwls.approximate_hessian + if HessianEvaluation(semwls) === ExactHessian J = -2*(σ₋'*semwls.V)' ∇²Σ_function!(∇²Σ, J, par) hessian .+= ∇²Σ @@ -246,9 +232,6 @@ function objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSing end end -objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle, has_meanstructure::Val{true}) = - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - ############################################################################################ ### Recommended methods ############################################################################################ diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl index cb42d9340..0a6837e67 100644 --- a/src/loss/constant/constant.jl +++ b/src/loss/constant/constant.jl @@ -25,7 +25,7 @@ Analytic gradients and hessians are available. ## Implementation Subtype of `SemLossFunction`. """ -struct SemConstant{C} <: SemLossFunction +struct SemConstant{C} <: SemLossFunction{ExactHessian} c::C end diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index f5f29cfb9..4badd462b 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -29,7 +29,7 @@ Analytic gradients and hessians are available. ## Implementation Subtype of `SemLossFunction`. """ -struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction +struct SemRidge{P, W1, W2, GT, HT} <: SemLossFunction{ExactHessian} α::P which::W1 which_H::W2 diff --git a/src/types.jl b/src/types.jl index e4e1ab2f1..4b671e5ed 100644 --- a/src/types.jl +++ b/src/types.jl @@ -10,8 +10,30 @@ abstract type AbstractSemSingle{O, I, L, D} <: AbstractSem end "Supertype for all collections of multiple SEMs" abstract type AbstractSemCollection <: AbstractSem end +"Meanstructure trait for `SemImply` subtypes" +abstract type MeanStructure end +"Indicates that `SemImply` subtype supports meanstructure" +struct HasMeanStructure <: MeanStructure end +"Indicates that `SemImply` subtype does not support meanstructure" +struct NoMeanStructure <: MeanStructure end + +# fallback implementation +MeanStructure(::Type{T}) where T = error("Objects of type $T do not support MeanStructure trait") +MeanStructure(semobj) = MeanStructure(typeof(semobj)) + +"Hessian Evaluation trait for `SemImply` and `SemLossFunction` subtypes" +abstract type HessianEvaluation end +struct ApproximateHessian <: HessianEvaluation end +struct ExactHessian <: HessianEvaluation end + +# fallback implementation +HessianEvaluation(::Type{T}) where T = error("Objects of type $T do not support HessianEvaluation trait") +HessianEvaluation(semobj) = HessianEvaluation(typeof(semobj)) + "Supertype for all loss functions of SEMs. If you want to implement a custom loss function, it should be a subtype of `SemLossFunction`." -abstract type SemLossFunction end +abstract type SemLossFunction{HE <: HessianEvaluation} end + +HessianEvaluation(::Type{<:SemLossFunction{HE}}) where HE <: HessianEvaluation = HE """ SemLoss(args...; loss_weights = nothing, ...) @@ -77,10 +99,13 @@ Computed model-implied values that should be compared with the observed data to e. g. the model implied covariance or mean. If you would like to implement a different notation, e.g. LISREL, you should implement a subtype of SemImply. """ -abstract type SemImply end +abstract type SemImply{MS <: MeanStructure, HE <: HessianEvaluation} end + +MeanStructure(::Type{<:SemImply{MS}}) where MS <: MeanStructure = MS +HessianEvaluation(::Type{<:SemImply{MS,HE}}) where {MS, HE <: MeanStructure} = HE "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." -abstract type SemImplySymbolic <: SemImply end +abstract type SemImplySymbolic{MS,HE} <: SemImply{MS,HE} end """ Sem(;observed = SemObservedData, imply = RAM, loss = SemML, optimizer = SemOptimizerOptim, kwargs...) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 7824efdb1..41d7d55bb 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -114,7 +114,7 @@ end # ML estimation - user defined loss function ############################################################################################ -struct UserSemML <: SemLossFunction end +struct UserSemML <: SemLossFunction{ExactHessian} end ############################################################################################ ### functors From 692d6dfde8ae407540432cc528d328bb1235d5f6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 12:14:33 -0700 Subject: [PATCH 050/174] RAMMatrices: option to keep zero constants --- src/frontend/specification/RAMMatrices.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 109c034f3..da44ea7e4 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -17,9 +17,10 @@ function ==(c1::RAMConstant, c2::RAMConstant) end function append_RAMConstants!(constants::AbstractVector{RAMConstant}, - mtx_name::Symbol, mtx::AbstractArray) + mtx_name::Symbol, mtx::AbstractArray; + skip_zeros::Bool = true) for (index, val) in pairs(mtx) - if isa(val, Number) && !iszero(val) + if isa(val, Number) && !(skip_zeros && iszero(val)) push!(constants, RAMConstant(mtx_name, index, val)) end end From 7f1d1715102928374a24fb96300ba3aa5ed2bc0b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 18:41:13 -0700 Subject: [PATCH 051/174] getindex(EnsParTable, i) instead of get_group() --- src/frontend/specification/EnsembleParameterTable.jl | 4 +--- src/frontend/specification/RAMMatrices.jl | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index c3d5929e3..8798a569a 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -102,9 +102,7 @@ end push!(partable::EnsembleParameterTable, d::Nothing, group) = nothing -# get group -------------------------------------------------------------------------------- - -get_group(partable::EnsembleParameterTable, group) = get_group(partable.tables, group) +Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] ############################################################################################ ### Update Partable from Fitted Model diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index da44ea7e4..5325c3677 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -348,7 +348,3 @@ function ==(mat1::RAMMatrices, mat2::RAMMatrices) (mat1.constants == mat2.constants) ) return res end - -function get_group(d::Dict, group) - return d[group] -end From f4411a166a1817cabab85a4b74121f4fb415c0d4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:14:05 -0700 Subject: [PATCH 052/174] matrix_gradient(): refactor * rename from get_matrix_derivative(): it's not a getter, and gradient is a better term * construct sparse matrix directly, which is much more efficient * parameters arg is not needed --- src/additional_functions/parameters.jl | 20 ++++++++------------ src/imply/RAM/generic.jl | 13 +++---------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl index 9a21c95a6..e9662e274 100644 --- a/src/additional_functions/parameters.jl +++ b/src/additional_functions/parameters.jl @@ -106,18 +106,14 @@ function check_constants(M) end -function get_matrix_derivative(M_indices, parameters, n_long) - - ∇M = [ - sparsevec( - M_indices[i], - ones(length(M_indices[i])), - n_long) for i in 1:length(parameters)] - - ∇M = reduce(hcat, ∇M) - - return ∇M - +# construct length(M)×length(parameters) sparse matrix of 1s at the positions, +# where the corresponding parameter occurs in the M matrix +function matrix_gradient(M_indices::ArrayParamsMap, + M_length::Integer) + rowval = reduce(vcat, M_indices) + colptr = pushfirst!(accumulate((ptr, M_ind) -> ptr + length(M_ind), M_indices, init=1), 1) + return SparseMatrixCSC(M_length, length(M_indices), + colptr, rowval, ones(length(rowval))) end # fill M with parameters diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 58afd3191..a44b80528 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -136,8 +136,8 @@ function RAM(; I_A = similar(A_pre) if gradient - ∇A = get_matrix_derivative(A_indices, parameters, n_var^2) - ∇S = get_matrix_derivative(S_indices, parameters, n_var^2) + ∇A = matrix_gradient(A_indices, n_var^2) + ∇S = matrix_gradient(S_indices, n_var^2) else ∇A = nothing ∇S = nothing @@ -146,15 +146,8 @@ function RAM(; # μ if meanstructure MS = HasMeanStructure - - if gradient - ∇M = get_matrix_derivative(M_indices, parameters, n_var) - else - ∇M = nothing - end - + ∇M = gradient ? matrix_gradient(M_indices, n_var) : nothing μ = zeros(n_obs) - else MS = NoMeanStructure M_indices = nothing From 204044a166b2283248ac4af6a2bae745efeefc18 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:15:53 -0700 Subject: [PATCH 053/174] optimize kron --- src/loss/ML/ML.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 0a9169bae..df9aa3f1b 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -228,7 +228,7 @@ function objective_hessian!( J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' ∇²Σ_function!(∇²Σ, J, par) # outer - H_outer = 2*kron(Σ⁻¹ΣₒΣ⁻¹, Σ⁻¹) - kron(Σ⁻¹, Σ⁻¹) + H_outer = kron(2Σ⁻¹ΣₒΣ⁻¹ - Σ⁻¹, Σ⁻¹) hessian = ∇Σ'*H_outer*∇Σ hessian .+= ∇²Σ end From 71bb58f5e394c7f93c0103267b8a3e87cffb8e64 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:26:41 -0700 Subject: [PATCH 054/174] ML: optimize C avoid transpose and * since sigma matrices are symmetric --- src/loss/ML/ML.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index df9aa3f1b..3a2734da1 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -388,7 +388,7 @@ function gradient!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) #mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - C = F⨉I_A⁻¹'*(I-Σₒ*Σ⁻¹)'*Σ⁻¹*F⨉I_A⁻¹ + C = F⨉I_A⁻¹'*(I-Σ⁻¹Σₒ)*Σ⁻¹*F⨉I_A⁻¹ gradient = 2vec(C*S*I_A⁻¹')'∇A + vec(C)'∇S if MeanStructure(imply(model)) === HasMeanStructure From 79616dd315103324691dbce744421452a20d2d12 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:30:23 -0700 Subject: [PATCH 055/174] refactor get_partition() * convert to param_range() which gets the range for a single matrix * use findfirst/findlast() instead of manual loop --- src/additional_functions/parameters.jl | 99 ++++---------------------- 1 file changed, 14 insertions(+), 85 deletions(-) diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl index e9662e274..731116f7c 100644 --- a/src/additional_functions/parameters.jl +++ b/src/additional_functions/parameters.jl @@ -128,92 +128,21 @@ function fill_matrix!(M::AbstractMatrix, M_indices::AbstractArrayParamsMap, return M end -function get_partition(A_indices, S_indices) - - n_par = length(A_indices) - - first_A = "a" - first_S = "a" - last_A = "a" - last_S = "a" - - for i in 1:n_par - if length(A_indices[i]) != 0 - first_A = i - break - end - end - - for i in 1:n_par - if length(S_indices[i]) != 0 - first_S = i - break - end - end - - for i in n_par+1 .- (1:n_par) - if length(A_indices[i]) != 0 - last_A = i - break - end - end - - for i in n_par+1 .- (1:n_par) - if length(S_indices[i]) != 0 - last_S = i - break - end - end - - for i in first_A:last_A - if length(A_indices[i]) == 0 - throw(ErrorException( - "Your parameter vector is not partitioned into directed and undirected effects")) - return nothing - end - end - - for i in first_S:last_S - if length(S_indices[i]) == 0 - throw(ErrorException( - "Your parameter vector is not partitioned into directed and undirected effects")) - return nothing +# range of parameters that are referenced in the matrix +function param_range(mtx_indices::AbstractArrayParamsMap) + + first_i = findfirst(!isempty, mtx_indices) + last_i = findlast(!isempty, mtx_indices) + + if !isnothing(first_i) && !isnothing(last_i) + for i in first_i:last_i + if isempty(mtx_indices[i]) + # TODO show which parameter is missing in which matrix + throw(ErrorException( + "Your parameter vector is not partitioned into directed and undirected effects")) + end end end - return first_A:last_A, first_S:last_S - + return first_i:last_i end - -function get_partition(M_indices) - - n_par = length(M_indices) - - first_M = "a" - last_M = "a" - - for i in 1:n_par - if length(M_indices[i]) != 0 - first_M = i - break - end - end - - for i in n_par+1 .- (1:n_par) - if length(M_indices[i]) != 0 - last_M = i - break - end - end - - for i in first_M:last_M - if length(M_indices[i]) == 0 - throw(ErrorException( - "Your parameter vector is not partitioned into directed, undirected and mean effects")) - return nothing - end - end - - return first_M:last_M - -end \ No newline at end of file From 64219f56e98f5ef626d088589320fe74a54b1466 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:32:28 -0700 Subject: [PATCH 056/174] remove_all_missing(): optimize avoid unnecessary allocations --- src/additional_functions/helper.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 6505de400..54c66d3e7 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -58,10 +58,10 @@ function F_one_person(imp_mean, meandiff, inverse, data, logdet) return F end -function remove_all_missing(data) +function remove_all_missing(data::AbstractMatrix) keep = Vector{Int64}() - for i = 1:size(data, 1) - if any(.!ismissing.(data[i, :])) + for (i, coldata) in zip(axes(data, 1), eachrow(data)) + if any(!ismissing, coldata) push!(keep, i) end end From ef8191975e43a54562364b782a6b8eeabf923bd1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:33:40 -0700 Subject: [PATCH 057/174] skipmissing_mean(): optimize * use eachcol() within comprehension --- src/additional_functions/helper.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 54c66d3e7..0db93ea68 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -43,11 +43,9 @@ function get_observed(rowind, data, semobserved; return observed_vec end -function skipmissing_mean(mat) - means = Vector{Float64}(undef, size(mat, 2)) - for i = 1:size(mat, 2) - @views means[i] = mean(skipmissing(mat[:,i])) - end +function skipmissing_mean(mat::AbstractMatrix) + means = [mean(skipmissing(coldata)) + for coldata in eachcol(mat)] return means end From af8a9e3605e2664a89befda8a576f5af4dfc8af2 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:37:33 -0700 Subject: [PATCH 058/174] get_observed(): refactor * rename to observed() since it is not a getter * change the signature to match Julia conventions --- src/additional_functions/helper.jl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 0db93ea68..d7da48c25 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -30,17 +30,16 @@ function semvec(observed, imply, loss, optimizer) return sem_vec end -function get_observed(rowind, data, semobserved; +# construct a vector of SemObserved objects +# for each specified data row +function observed(::Type{T}, data, rowinds; args = (), - kwargs = NamedTuple()) - observed_vec = Vector{semobserved}(undef, length(rowind)) - for i in 1:length(rowind) - observed_vec[i] = semobserved( - args...; - data = Matrix(data[rowind[i], :]), - kwargs...) - end - return observed_vec + kwargs = NamedTuple()) where T <: SemObserved + return T[ + T(args...; + data = Matrix(view(data, row, :)), + kwargs...) + for row in rowinds] end function skipmissing_mean(mat::AbstractMatrix) From fa1c45351b3c443ffe970a47718d1d3e773c8521 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:10:30 -0700 Subject: [PATCH 059/174] fix ridge eval --- src/loss/regularization/ridge.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 4badd462b..17d0b8c23 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -57,11 +57,10 @@ function SemRidge(; which_ridge = get_identifier_indices(which_ridge, imply) end end - which = [CartesianIndex(x) for x in which_ridge] which_H = [CartesianIndex(x, x) for x in which_ridge] return SemRidge( α_ridge, - which, + which_ridge, which_H, zeros(parameter_type, nparams), @@ -72,15 +71,15 @@ end ### methods ############################################################################################ -objective!(ridge::SemRidge, par, model) = @views ridge.α*sum(x -> x^2, par[ridge.which]) +objective!(ridge::SemRidge, par, model) = @views ridge.α*sum(abs2, par[ridge.which]) function gradient!(ridge::SemRidge, par, model) - @views ridge.gradient[ridge.which] .= 2*ridge.α*par[ridge.which] + @views ridge.gradient[ridge.which] .= (2*ridge.α)*par[ridge.which] return ridge.gradient end function hessian!(ridge::SemRidge, par, model) - @views @. ridge.hessian[ridge.which_H] += ridge.α*2.0 + @views @. ridge.hessian[ridge.which_H] .= 2*ridge.α return ridge.hessian end From a4dba9b96e5c328044900695ef67d6fd347ef374 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 20:35:06 -0700 Subject: [PATCH 060/174] obj/grad/hess: refactor evaluation API the intent of this commit is to refactor the API for objective, gradient and hessian evaluation, such that the evaluation code does not have to be duplicates across functions that calculate different combinations of those functions * introduce EvaluationTargets class that handles selection of what to evaluate * add evaluate!(EvalTargets, ...) methods for loss and imply objs that evaluate only what is required * objective!(), obj_grad!() etc calls are just a wrapper of evaluate!() with proper targets --- src/frontend/fit/standard_errors/hessian.jl | 2 +- src/imply/RAM/generic.jl | 113 +---- src/imply/RAM/symbolic.jl | 43 +- src/imply/empty.jl | 4 +- src/loss/ML/FIML.jl | 84 ++-- src/loss/ML/ML.jl | 443 +++++--------------- src/loss/WLS/WLS.jl | 169 ++------ src/loss/constant/constant.jl | 6 +- src/loss/regularization/ridge.jl | 14 +- src/objective_gradient_hessian.jl | 410 ++++++------------ test/examples/multigroup/build_models.jl | 27 +- 11 files changed, 327 insertions(+), 988 deletions(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 0603b3ad4..f713920d7 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -18,7 +18,7 @@ function se_hessian(sem_fit::SemFit; hessian = :finitediff) hessian!(H, sem_fit.model, sem_fit.solution) elseif hessian == :finitediff H = FiniteDiff.finite_difference_hessian( - Base.Fix1(objective!, sem_fit.model), + p -> evaluate!(eltype(sem_fit.solution), nothing, nothing, fit.model, p), sem_fit.solution ) elseif hessian == :optimizer diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index a44b80528..f2a3ca48b 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -95,19 +95,16 @@ end ### Constructors ############################################################################################ - RAM{MS}(args...) where MS <: MeanStructure = RAM{MS, map(typeof, args)...}(args...) function RAM(; specification::SemSpecification, #vech = false, - gradient = true, + gradient_required = true, meanstructure = false, kwargs...) ram_matrices = convert(RAMMatrices, specification) - identifier = StructuralEquationModels.identifier(ram_matrices) - # get dimensions of the model n_par = nparams(ram_matrices) @@ -135,7 +132,7 @@ function RAM(; F⨉I_A⁻¹S = zeros(n_obs, n_var) I_A = similar(A_pre) - if gradient + if gradient_required ∇A = matrix_gradient(A_indices, n_var^2) ∇S = matrix_gradient(S_indices, n_var^2) else @@ -179,7 +176,7 @@ function RAM(; ∇S, ∇M, - identifier + identifier(ram_matrices) ) end @@ -187,14 +184,13 @@ end ### methods ############################################################################################ -# objective and gradient -function objective!(imply::RAM, parameters, model) - +function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, parameters) + fill_A_S_M!( imply.A, imply.S, imply.M, - imply.A_indices, + imply.A_indices, imply.S_indices, imply.M_indices, parameters) @@ -205,64 +201,23 @@ function objective!(imply::RAM, parameters, model) end end - copyto!(imply.F⨉I_A⁻¹, imply.F) - rdiv!(imply.F⨉I_A⁻¹, factorize(imply.I_A)) - - Σ_RAM!( - imply.Σ, - imply.F⨉I_A⁻¹, - imply.S, - imply.F⨉I_A⁻¹S) - - if MeanStructure(imply) === HasMeanStructure - μ_RAM!(imply.μ, imply.F⨉I_A⁻¹, imply.M) - end - -end - -function gradient!(imply::RAM, parameters, model::AbstractSemSingle) - - fill_A_S_M!( - imply.A, - imply.S, - imply.M, - imply.A_indices, - imply.S_indices, - imply.M_indices, - parameters) - - @inbounds for (j, I_Aj, Aj) in zip(axes(imply.A, 2), eachcol(imply.I_A), eachcol(imply.A)) - for i in axes(imply.A, 1) - I_Aj[i] = ifelse(i == j, 1, 0) - Aj[i] - end + if is_gradient_required(targets) || is_hessian_required(targets) + imply.I_A⁻¹ = LinearAlgebra.inv!(factorize(imply.I_A)) + mul!(imply.F⨉I_A⁻¹, imply.F, imply.I_A⁻¹) + else + copyto!(imply.F⨉I_A⁻¹, imply.F) + rdiv!(imply.F⨉I_A⁻¹, factorize(imply.I_A)) end - imply.I_A⁻¹ = LinearAlgebra.inv!(factorize(imply.I_A)) - mul!(imply.F⨉I_A⁻¹, imply.F, imply.I_A⁻¹) - - Σ_RAM!( - imply.Σ, - imply.F⨉I_A⁻¹, - imply.S, - imply.F⨉I_A⁻¹S) + mul!(imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹, imply.S) + mul!(imply.Σ, imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹') if MeanStructure(imply) === HasMeanStructure - μ_RAM!(imply.μ, imply.F⨉I_A⁻¹, imply.M) + mul!(imply.μ, imply.F⨉I_A⁻¹, imply.M) end end -hessian!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model) -objective_gradient!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model) -objective_hessian!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model) -gradient_hessian!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model) -objective_gradient_hessian!(imply::RAM, par, model::AbstractSemSingle) = - gradient!(imply, par, model) - ############################################################################################ ### Recommended methods ############################################################################################ @@ -278,48 +233,10 @@ function update_observed(imply::RAM, observed::SemObserved; kwargs...) end end -############################################################################################ -### additional methods -############################################################################################ - -Σ(imply::RAM) = imply.Σ -μ(imply::RAM) = imply.μ - -A(imply::RAM) = imply.A -S(imply::RAM) = imply.S -F(imply::RAM) = imply.F -M(imply::RAM) = imply.M - -∇A(imply::RAM) = imply.∇A -∇S(imply::RAM) = imply.∇S -∇M(imply::RAM) = imply.∇M - -A_indices(imply::RAM) = imply.A_indices -S_indices(imply::RAM) = imply.S_indices -M_indices(imply::RAM) = imply.M_indices - -F⨉I_A⁻¹(imply::RAM) = imply.F⨉I_A⁻¹ -F⨉I_A⁻¹S(imply::RAM) = imply.F⨉I_A⁻¹S -I_A(imply::RAM) = imply.I_A -I_A⁻¹(imply::RAM) = imply.I_A⁻¹ # only for gradient available! - -has_meanstructure(imply::RAM) = imply.has_meanstructure - -ram_matrices(imply::RAM) = imply.ram_matrices - ############################################################################################ ### additional functions ############################################################################################ -function Σ_RAM!(Σ, F⨉I_A⁻¹, S, pre2) - mul!(pre2, F⨉I_A⁻¹, S) - mul!(Σ, pre2, F⨉I_A⁻¹') -end - -function μ_RAM!(μ, F⨉I_A⁻¹, M) - mul!(μ, F⨉I_A⁻¹, M) -end - function check_acyclic(A_pre, n_par, A_indices) # fill copy of A-matrix with random parameters A_rand = copy(A_pre) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index da1e7e595..5328cc9b8 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -201,30 +201,20 @@ end ### objective, gradient, hessian ############################################################################################ -# objective -function objective!(imply::RAMSymbolic, par, model) +function update!(targets::EvaluationTargets, imply::RAMSymbolic, model::AbstractSemSingle, par) imply.Σ_function(imply.Σ, par) if MeanStructure(imply) === HasMeanStructure imply.μ_function(imply.μ, par) end -end -# gradient -function gradient!(imply::RAMSymbolic, par, model) - objective!(imply, par, model, imply.has_meanstructure) - imply.∇Σ_function(imply.∇Σ, par) - if MeanStructure(imply) === HasMeanStructure - imply.∇μ_function(imply.∇μ, par) + if is_gradient_required(targets) || is_hessian_required(targets) + imply.∇Σ_function(imply.∇Σ, par) + if MeanStructure(imply) === HasMeanStructure + imply.∇μ_function(imply.∇μ, par) + end end end -# other methods -hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) -objective_gradient!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) -objective_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) -gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) -objective_gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, par, model) - ############################################################################################ ### Recommended methods ############################################################################################ @@ -232,7 +222,7 @@ objective_gradient_hessian!(imply::RAMSymbolic, par, model) = gradient!(imply, p identifier(imply::RAMSymbolic) = imply.identifier nparams(imply::RAMSymbolic) = nparams(imply.ram_matrices) -function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) +function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) if Int(n_man(observed)) == size(imply.Σ, 1) return imply else @@ -240,25 +230,6 @@ function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) end end -############################################################################################ -### additional methods -############################################################################################ - -Σ(imply::RAMSymbolic) = imply.Σ -∇Σ(imply::RAMSymbolic) = imply.∇Σ -∇²Σ(imply::RAMSymbolic) = imply.∇²Σ - -μ(imply::RAMSymbolic) = imply.μ -∇μ(imply::RAMSymbolic) = imply.∇μ - -Σ_function(imply::RAMSymbolic) = imply.Σ_function -∇Σ_function(imply::RAMSymbolic) = imply.∇Σ_function -∇²Σ_function(imply::RAMSymbolic) = imply.∇²Σ_function - -has_meanstructure(imply::RAMSymbolic) = imply.has_meanstructure - -ram_matrices(imply::RAMSymbolic) = imply.ram_matrices - ############################################################################################ ### additional functions ############################################################################################ diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 9a0231536..ce4664a4a 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -50,9 +50,7 @@ end ### methods ############################################################################################ -objective!(imply::ImplyEmpty, par, model) = nothing -gradient!(imply::ImplyEmpty, par, model) = nothing -hessian!(imply::ImplyEmpty, par, model) = nothing +update!(targets::EvaluationTargets, imply::ImplyEmpty, par, model) = nothing ############################################################################################ ### Recommended methods diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index c49b8f916..0c9e4071b 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -84,38 +84,25 @@ end ### methods ############################################################################################ -function objective!(semfiml::SemFIML, parameters, model) +function evaluate!(objective, gradient, hessian, + semfiml::SemFIML, implied::SemImply, model::AbstractSemSingle, parameters) - if !check_fiml(semfiml, model) return non_posdef_return(parameters) end + isnothing(hessian) || error("Hessian not implemented for FIML") - prepare_SemFIML!(semfiml, model) - - objective = F_FIML(rows(observed(model)), semfiml, model, parameters) - return objective/n_obs(observed(model)) -end - -function gradient!(semfiml::SemFIML, parameters, model) - - if !check_fiml(semfiml, model) return ones(eltype(parameters), size(parameters)) end - - prepare_SemFIML!(semfiml, model) - - gradient = ∇F_FIML(rows(observed(model)), semfiml, model)/n_obs(observed(model)) - return gradient -end - -function objective_gradient!(semfiml::SemFIML, parameters, model) - - if !check_fiml(semfiml, model) - return non_posdef_return(parameters), ones(eltype(parameters), size(parameters)) + if !check_fiml(semfiml, model) + isnothing(objective) || (objective = non_posdef_return(parameters)) + isnothing(gradient) || fill!(gradient, 1) + return objective end prepare_SemFIML!(semfiml, model) - objective = F_FIML(rows(observed(model)), semfiml, model, parameters)/n_obs(observed(model)) - gradient = ∇F_FIML(rows(observed(model)), semfiml, model)/n_obs(observed(model)) + scale = inv(n_obs(observed(model))) + obs_rows = rows(observed(model)) + isnothing(objective) || (objective = scale*F_FIML(obs_rows, semfiml, model, parameters)) + isnothing(gradient) || (∇F_FIML!(gradient, obs_rows, semfiml, model); gradient .*= scale) - return objective, gradient + return objective end ############################################################################################ @@ -130,13 +117,11 @@ update_observed(lossfun::SemFIML, observed::SemObserved; kwargs...) = ############################################################################################ function F_one_pattern(meandiff, inverse, obs_cov, logdet, N) - F = logdet - F += meandiff'*inverse*meandiff + F = logdet + dot(meandiff, inverse, meandiff) if N > one(N) F += dot(obs_cov, inverse) end - F = N*F - return F + return F*N end function ∇F_one_pattern(μ_diff, Σ⁻¹, S, pattern, ∇ind, N, Jμ, JΣ, model) @@ -153,26 +138,25 @@ function ∇F_one_pattern(μ_diff, Σ⁻¹, S, pattern, ∇ind, N, Jμ, JΣ, mod end -function ∇F_fiml_outer(JΣ, Jμ, imply::SemImplySymbolic, model, semfiml) - G = transpose(JΣ'*∇Σ(imply)-Jμ'*∇μ(imply)) - return G +function ∇F_fiml_outer!(G, JΣ, Jμ, imply::SemImplySymbolic, model, semfiml) + mul!(G, imply.∇Σ', JΣ) # should be transposed + G .-= imply.∇μ' * Jμ end -function ∇F_fiml_outer(JΣ, Jμ, imply, model, semfiml) +function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) - Iₙ = sparse(1.0I, size(A(imply))...) - P = kron(F⨉I_A⁻¹(imply), F⨉I_A⁻¹(imply)) - Q = kron(S(imply)*I_A⁻¹(imply)', Iₙ) + Iₙ = sparse(1.0I, size(imply.A)...) + P = kron(imply.F⨉I_A⁻¹, imply.F⨉I_A⁻¹) + Q = kron(imply.S*imply.I_A⁻¹', Iₙ) #commutation_matrix_pre_square_add!(Q, Q) Q2 = commutation_matrix_pre_square(Q, semfiml.commutation_indices) - ∇Σ = P*(∇S(imply) + (Q+Q2)*∇A(imply)) - - ∇μ = F⨉I_A⁻¹(imply)*∇M(imply) + kron((I_A⁻¹(imply)*M(imply))', F⨉I_A⁻¹(imply))*∇A(imply) + ∇Σ = P*(imply.∇S + (Q+Q2)*imply.∇A) - G = transpose(JΣ'*∇Σ-Jμ'*∇μ) + ∇μ = imply.F⨉I_A⁻¹*imply.∇M + kron((imply.I_A⁻¹*imply.M)', imply.F⨉I_A⁻¹)*imply.∇A - return G + mul!(G, ∇Σ', JΣ) # actually transposed + G .-= ∇μ' * Jμ end function F_FIML(rows, semfiml, model, parameters) @@ -188,7 +172,7 @@ function F_FIML(rows, semfiml, model, parameters) return F end -function ∇F_FIML(rows, semfiml, model) +function ∇F_FIML!(G, rows, semfiml, model) Jμ = zeros(Int64(n_man(model))) JΣ = zeros(Int64(n_man(model)^2)) @@ -204,7 +188,7 @@ function ∇F_FIML(rows, semfiml, model) JΣ, model) end - return ∇F_fiml_outer(JΣ, Jμ, imply(model), model, semfiml) + ∇F_fiml_outer!(G, JΣ, Jμ, imply(model), model, semfiml) end function prepare_SemFIML!(semfiml, model) @@ -232,13 +216,13 @@ function copy_per_pattern!(inverses, source_inverses, means, source_means, patte end copy_per_pattern!( - semfiml, - model::M where {M <: AbstractSem}) = + semfiml, + model::M where {M <: AbstractSem}) = copy_per_pattern!( - semfiml.inverses, - Σ(imply(model)), - semfiml.imp_mean, - μ(imply(model)), + semfiml.inverses, + imply(model).Σ, + semfiml.imp_mean, + imply(model).μ, patterns(observed(model))) function batch_cholesky!(semfiml, model) @@ -250,7 +234,7 @@ function batch_cholesky!(semfiml, model) end function check_fiml(semfiml, model) - copyto!(semfiml.imp_inv, Σ(imply(model))) + copyto!(semfiml.imp_inv, imply(model).Σ) a = cholesky!(Symmetric(semfiml.imp_inv); check = false) return isposdef(a) end diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 3a2734da1..7681493ca 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -56,391 +56,145 @@ end ### objective, gradient, hessian methods ############################################################################################ -# dispatch for SemImply -objective!(semml::SemML, par, model::AbstractSemSingle) = - objective!(semml, par, model, imply(model)) -gradient!(semml::SemML, par, model::AbstractSemSingle) = - gradient!(semml, par, model, imply(model)) -hessian!(semml::SemML, par, model::AbstractSemSingle) = - hessian!(semml, par, model, imply(model)) -objective_gradient!(semml::SemML, par, model::AbstractSemSingle) = - objective_gradient!(semml, par, model, imply(model)) -objective_hessian!(semml::SemML, par, model::AbstractSemSingle) = - objective_hessian!(semml, par, model, imply(model)) -gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = - gradient_hessian!(semml, par, model, imply(model)) -objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle) = - objective_gradient_hessian!(semml, par, model, imply(model)) - ############################################################################################ ### Symbolic Imply Types -function objective!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::SemImplySymbolic) - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), μ = μ(imply(model)), μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return non_posdef_return(par) - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - #mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - return ld + dot(Σ⁻¹, Σₒ) + dot(μ₋, Σ⁻¹, μ₋) - else - return ld + dot(Σ⁻¹, Σₒ) - end - end -end - -function gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::SemImplySymbolic) - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), ∇Σ = ∇Σ(imply(model)), - μ = μ(imply(model)), ∇μ = ∇μ(imply(model)), μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return ones(eltype(par), size(par)) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) +function evaluate!( + objective, gradient, hessian, + semml::SemML, + implied::SemImplySymbolic, + model::AbstractSemSingle, + par) - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ - gradient = vec(Σ⁻¹*(I - Σₒ*Σ⁻¹ - μ₋*μ₋ᵀΣ⁻¹))'*∇Σ - 2*μ₋ᵀΣ⁻¹*∇μ - else - gradient = (vec(Σ⁻¹)-vec(Σ⁻¹Σₒ*Σ⁻¹))'*∇Σ - end - return gradient' - end -end - -function hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::SemImplySymbolic) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of ML + meanstructure is not available")) + if !isnothing(hessian) + (MeanStructure(implied) === HasMeanStructure) && + throw(DomainError(H, "hessian of ML + meanstructure is not available")) end - let Σ = Σ(imply(model)), ∇Σ = ∇Σ(imply(model)), Σₒ = obs_cov(observed(model)), - Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), - ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) + Σ = implied.Σ + Σₒ = obs_cov(observed(model)) + Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ + Σ⁻¹ = semml.Σ⁻¹ copyto!(Σ⁻¹, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || - return diagm(fill(one(eltype(par)), length(par))) + if !isposdef(Σ_chol) + #@warn "∑⁻¹ is not positive definite" + isnothing(objective) || (objective = non_posdef_return(par)) + isnothing(gradient) || fill!(gradient, 1) + isnothing(hessian) || copyto!(hessian, I) + return objective + end + ld = logdet(Σ_chol) Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - - if HessianEvaluation(semml) === ApproximateHessian - hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ - else - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + isnothing(objective) || (objective = ld + tr(Σ⁻¹Σₒ)) + + if MeanStructure(implied) === HasMeanStructure + μ = implied.μ + μₒ = obs_mean(observed(model)) + μ₋ = μₒ - μ + + isnothing(objective) || (objective += dot(μ₋, Σ⁻¹, μ₋)) + if !isnothing(gradient) + ∇Σ = implied.∇Σ + ∇μ = implied.∇μ + μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ + gradient .= (vec(Σ⁻¹*(I - Σₒ*Σ⁻¹ - μ₋*μ₋ᵀΣ⁻¹))' * ∇Σ)' + gradient .-= (2*μ₋ᵀΣ⁻¹*∇μ)' + end + elseif !isnothing(gradient) || !isnothing(hessian) + ∇Σ = implied.∇Σ Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ - # inner J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' - ∇²Σ_function!(∇²Σ, J, par) - # outer - H_outer = 2*kron(Σ⁻¹ΣₒΣ⁻¹, Σ⁻¹) - kron(Σ⁻¹, Σ⁻¹) - hessian = ∇Σ'*H_outer*∇Σ - hessian .+= ∇²Σ - end - - return hessian - end -end - -function objective_gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::SemImplySymbolic) - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), μ = μ(imply(model)), μₒ = obs_mean(observed(model)), - ∇Σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - if !isposdef(Σ_chol) - return non_posdef_return(par), ones(eltype(par), size(par)) - else - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ - - objective = ld + tr(Σ⁻¹Σₒ) + dot(μ₋, Σ⁻¹, μ₋) - gradient = vec(Σ⁻¹*(I - Σₒ*Σ⁻¹ - μ₋*μ₋ᵀΣ⁻¹))'*∇Σ - 2*μ₋ᵀΣ⁻¹*∇μ - else - objective = ld + tr(Σ⁻¹Σₒ) - gradient = (vec(Σ⁻¹)-vec(Σ⁻¹Σₒ*Σ⁻¹))'*∇Σ - end - return objective, gradient' + if !isnothing(gradient) + gradient .= (J*∇Σ)' end - end -end - -function objective_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::SemImplySymbolic) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of ML + meanstructure is not available")) - end - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), ∇Σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - if !isposdef(Σ_chol) - return non_posdef_return(par), diagm(fill(one(eltype(par)), length(par))) - else - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - objective = ld + tr(Σ⁻¹Σₒ) - + if !isnothing(hessian) if HessianEvaluation(semml) === ApproximateHessian - hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ + mul!(hessian, 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹), ∇Σ) else - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ + ∇²Σ_function! = implied.∇²Σ_function + ∇²Σ = implied.∇²Σ # inner - J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' ∇²Σ_function!(∇²Σ, J, par) # outer H_outer = kron(2Σ⁻¹ΣₒΣ⁻¹ - Σ⁻¹, Σ⁻¹) - hessian = ∇Σ'*H_outer*∇Σ + mul!(hessian, ∇Σ'*H_outer, ∇Σ) hessian .+= ∇²Σ end - - return objective, hessian end end -end - -function gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::SemImplySymbolic) - - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of ML + meanstructure is not available")) - end - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), ∇Σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || - return ones(eltype(par), size(par)), diagm(fill(one(eltype(par)), length(par))) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ - - J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' - gradient = J*∇Σ - - if HessianEvaluation(semml) === ApproximateHessian - hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ - else - # inner - ∇²Σ_function!(∇²Σ, J, par) - # outer - H_outer = 2*kron(Σ⁻¹ΣₒΣ⁻¹, Σ⁻¹) - kron(Σ⁻¹, Σ⁻¹) - hessian = ∇Σ'*H_outer*∇Σ - hessian .+= ∇²Σ - end - - return gradient', hessian - end -end - -function objective_gradient_hessian!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::SemImplySymbolic) - - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of ML + meanstructure is not available")) - end - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), ∇Σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - if !isposdef(Σ_chol) - objective = non_posdef_return(par) - gradient = ones(eltype(par), size(par)) - hessian = diagm(fill(one(eltype(par)), length(par))) - return objective, gradient, hessian - end - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - objective = ld + tr(Σ⁻¹Σₒ) - - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ - - J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' - gradient = J*∇Σ - - if HessianEvaluation(semml) == ApproximateHessian - hessian = 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹)*∇Σ - else - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ - # inner - ∇²Σ_function!(∇²Σ, J, par) - # outer - H_outer = 2*kron(Σ⁻¹ΣₒΣ⁻¹, Σ⁻¹) - kron(Σ⁻¹, Σ⁻¹) - hessian = ∇Σ'*H_outer*∇Σ - hessian .+= ∇²Σ - end - - return objective, gradient', hessian - end + return objective end ############################################################################################ ### Non-Symbolic Imply Types -# no hessians ------------------------------------------------------------------------------ +function evaluate!( + objective, gradient, hessian, + semml::SemML, + implied::RAM, + model::AbstractSemSingle, + par) -function hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) -end - -function objective_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) -end - -function gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) -end - -function objective_gradient_hessian!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - throw(DomainError(H, "hessian of ML + non-symbolic imply type is not available")) -end + if !isnothing(hessian) + error("hessian of ML + non-symbolic imply type is not available") + end -# objective, gradient ---------------------------------------------------------------------- + Σ = implied.Σ + Σₒ = obs_cov(observed(model)) + Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ + Σ⁻¹ = semml.Σ⁻¹ -function objective!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::RAM) - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), μ = μ(imply(model)), μₒ = obs_mean(observed(model)) + copyto!(Σ⁻¹, Σ) + Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) + if !isposdef(Σ_chol) + #@warn "Σ⁻¹ is not positive definite" + isnothing(objective) || (objective = non_posdef_return(par)) + isnothing(gradient) || fill!(gradient, 1) + isnothing(hessian) || copyto!(hessian, I) + return objective + end + ld = logdet(Σ_chol) + Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) + mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return non_posdef_return(par) - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + if !isnothing(objective) + objective = ld + tr(Σ⁻¹Σₒ) - if MeanStructure(imply(model)) === HasMeanStructure + if MeanStructure(implied) === HasMeanStructure + μ = implied.μ + μₒ = obs_mean(observed(model)) μ₋ = μₒ - μ - return ld + tr(Σ⁻¹Σₒ) + dot(μ₋, Σ⁻¹, μ₋) - else - return ld + tr(Σ⁻¹Σₒ) + objective += dot(μ₋, Σ⁻¹, μ₋) end end -end - -function gradient!(semml::SemML, par, model::AbstractSemSingle, imp::RAM) - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), - Σ⁻¹ = Σ⁻¹(semml), S = S(imply(model)), M = M(imply(model)), - F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), I_A⁻¹ = I_A⁻¹(imply(model)), - ∇A = ∇A(imply(model)), ∇S = ∇S(imply(model)), ∇M = ∇M(imply(model)), - μ = μ(imply(model)), μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - isposdef(Σ_chol) || return ones(eltype(par), size(par)) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - #mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + if !isnothing(gradient) + S = implied.S + F⨉I_A⁻¹ = implied.F⨉I_A⁻¹ + I_A⁻¹ = implied.I_A⁻¹ + ∇A = implied.∇A + ∇S = implied.∇S C = F⨉I_A⁻¹'*(I-Σ⁻¹Σₒ)*Σ⁻¹*F⨉I_A⁻¹ - gradient = 2vec(C*S*I_A⁻¹')'∇A + vec(C)'∇S + gradᵀ = 2vec(C*S*I_A⁻¹')'∇A + vec(C)'∇S - if MeanStructure(imply(model)) === HasMeanStructure + if MeanStructure(implied) === HasMeanStructure + μ = implied.μ + μₒ = obs_mean(observed(model)) + ∇M = implied.∇M + M = implied.M μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ - - gradient .+= -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S + gradᵀ .+= -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S end - - return gradient' + copyto!(gradient, gradᵀ') end -end -function objective_gradient!( - semml::SemML, - par, - model::AbstractSemSingle, - imp::RAM) - - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)), Σ⁻¹Σₒ = Σ⁻¹Σₒ(semml), Σ⁻¹ = Σ⁻¹(semml), - S = S(imply(model)), M = M(imply(model)), F⨉I_A⁻¹ = F⨉I_A⁻¹(imply(model)), I_A⁻¹ = I_A⁻¹(imply(model)), - ∇A = ∇A(imply(model)), ∇S = ∇S(imply(model)), ∇M = ∇M(imply(model)), - μ = μ(imply(model)), μₒ = obs_mean(observed(model)) - - copyto!(Σ⁻¹, Σ) - Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) - if !isposdef(Σ_chol) - objective = non_posdef_return(par) - gradient = ones(eltype(par), size(par)) - return objective, gradient - else - ld = logdet(Σ_chol) - Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) - objective = ld + tr(Σ⁻¹Σₒ) - - C = F⨉I_A⁻¹'*(I-Σₒ*Σ⁻¹)'*Σ⁻¹*F⨉I_A⁻¹ - gradient = 2vec(C*S*I_A⁻¹')'∇A + vec(C)'∇S - - if MeanStructure(semml) === HasMeanStructure - μ₋ = μₒ - μ - objective += dot(μ₋, Σ⁻¹, μ₋) - - μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ - k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ - gradient .+= -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S - end - - return objective, gradient' - end - end + return objective end ############################################################################################ @@ -459,8 +213,8 @@ end ### recommended methods ############################################################################################ -update_observed(lossfun::SemML, observed::SemObservedMissing; kwargs...) = - throw(ArgumentError("ML estimation does not work with missing data - use FIML instead")) +update_observed(lossfun::SemML, observed::SemObservedMissing; kwargs...) = + error("ML estimation does not work with missing data - use FIML instead") function update_observed(lossfun::SemML, observed::SemObserved; kwargs...) if size(lossfun.Σ⁻¹) == size(obs_cov(observed)) @@ -469,10 +223,3 @@ function update_observed(lossfun::SemML, observed::SemObserved; kwargs...) return SemML(;observed = observed, kwargs...) end end - -############################################################################################ -### additional methods -############################################################################################ - -Σ⁻¹(semml::SemML) = semml.Σ⁻¹ -Σ⁻¹Σₒ(semml::SemML) = semml.Σ⁻¹Σₒ \ No newline at end of file diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 03afad85b..45891a226 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -72,7 +72,7 @@ function SemWLS(;observed, wls_weight_matrix = nothing, wls_weight_matrix_mean = else wls_weight_matrix_mean = nothing end - HE = approximate_hessian ? ApproximateHessian : AnalyticHessian + HE = approximate_hessian ? ApproximateHessian : ExactHessian return SemWLS{HE}( wls_weight_matrix, @@ -85,151 +85,50 @@ end ### methods ############################################################################ -function objective!(semwls::SemWLS, par, model::AbstractSemSingle) - - let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, μₒ = obs_mean(observed(model)), V = semwls.V, V_μ = semwls.V_μ, - - σ₋ = σₒ - σ - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - return dot(σ₋, V, σ₋) + dot(μ₋, V_μ, μ₋) - else - return dot(σ₋, V, σ₋) - end - end -end - -function gradient!(semwls::SemWLS, par, model::AbstractSemSingle) - - let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, - μₒ = obs_mean(observed(model)), V = semwls.V, V_μ = semwls.V_μ, - ∇σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)) - - σ₋ = σₒ - σ - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - return -2*(σ₋'*V*∇σ + μ₋'*V_μ*∇μ)' - else - return -2*(σ₋'*V*∇σ)' - end +function evaluate!(objective, gradient, hessian, + semwls::SemWLS, implied::SemImplySymbolic, model::AbstractSemSingle, par) + if !isnothing(hessian) && (MeanStructure(implied) === HasMeanStructure) + error("hessian of WLS with meanstructure is not available") end -end - -function hessian!(semwls::SemWLS, par, model::AbstractSemSingle) - let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, - ∇σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) + V = semwls.V + ∇σ = implied.∇Σ - σ₋ = σₒ - σ + σ = implied.Σ + σₒ = semwls.σₒ + σ₋ = σₒ - σ - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - else - hessian = 2*∇σ'*V*∇σ - if HessianEvaluation(semwls) === ExactHessian - J = -2*(σ₋'*semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) - hessian .+= ∇²Σ - end - return hessian + isnothing(objective) || (objective = dot(σ₋, V, σ₋)) + if !isnothing(gradient) + if issparse(∇σ) + gradient .= (σ₋'*V*∇σ)' + else # save one allocation + mul!(gradient, σ₋'*V, ∇σ) # actually transposed, but should be fine for vectors end + gradient .*= -2 end -end - -function objective_gradient!(semwls::SemWLS, par, model::AbstractSemSingle) - - let σ = Σ(imply(model)), μ = μ(imply(model)), σₒ = semwls.σₒ, - μₒ = obs_mean(observed(model)), V = semwls.V, V_μ = semwls.V_μ, - ∇σ = ∇Σ(imply(model)), ∇μ = ∇μ(imply(model)) - - σ₋ = σₒ - σ - - if MeanStructure(imply(model)) === HasMeanStructure - μ₋ = μₒ - μ - objective = dot(σ₋, V, σ₋) + dot(μ₋, V_μ, μ₋) - gradient = -2*(σ₋'*V*∇σ + μ₋'*V_μ*∇μ)' - return objective, gradient - else - objective = dot(σ₋, V, σ₋) - gradient = -2*(σ₋'*V*∇σ)' - return objective, gradient - end - end -end - -function objective_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) + isnothing(hessian) || (mul!(hessian, ∇σ'*V, ∇σ); hessian .*= 2) + if !isnothing(hessian) && (HessianEvaluation(semwls) === ExactHessian) + ∇²Σ_function! = implied.∇²Σ_function + ∇²Σ = implied.∇²Σ + J = -2*(σ₋'*semwls.V)' + ∇²Σ_function!(∇²Σ, J, par) + hessian .+= ∇²Σ end - - let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, - ∇σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - - σ₋ = σₒ - σ - - objective = dot(σ₋, V, σ₋) - - hessian = 2*∇σ'*V*∇σ - if HessianEvaluation(semwls) === ExactHessian - J = -2*(σ₋'*semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) - hessian .+= ∇²Σ + if MeanStructure(implied) === HasMeanStructure + μ = implied.μ + μₒ = obs_mean(observed(model)) + μ₋ = μₒ - μ + V_μ = semwls.V_μ + if !isnothing(objective) + objective += dot(μ₋, V_μ, μ₋) end - - return objective, hessian - end -end - -function gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - end - - let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, - ∇σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - - σ₋ = σₒ - σ - - gradient = -2*(σ₋'*V*∇σ)' - - hessian = 2*∇σ'*V*∇σ - if HessianEvaluation(semwls) === ExactHessian - J = -2*(σ₋'*semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) - hessian .+= ∇²Σ + if !isnothing(gradient) + gradient .-= 2*(μ₋'*V_μ*implied.∇μ)' end - - return gradient, hessian end -end - -function objective_gradient_hessian!(semwls::SemWLS, par, model::AbstractSemSingle) - if MeanStructure(imply(model)) === HasMeanStructure - throw(DomainError(H, "hessian of WLS with meanstructure is not available")) - end - - let σ = Σ(imply(model)), σₒ = semwls.σₒ, V = semwls.V, - ∇σ = ∇Σ(imply(model)), - ∇²Σ_function! = ∇²Σ_function(imply(model)), ∇²Σ = ∇²Σ(imply(model)) - - σ₋ = σₒ - σ - - objective = dot(σ₋, V, σ₋) - gradient = -2*(σ₋'*V*∇σ)' - hessian = 2*∇σ'*V*∇σ - if HessianEvaluation(semwls) === ExactHessian - J = -2*(σ₋'*semwls.V)' - ∇²Σ_function!(∇²Σ, J, par) - hessian .+= ∇²Σ - end - return objective, gradient, hessian - end + return objective end ############################################################################################ diff --git a/src/loss/constant/constant.jl b/src/loss/constant/constant.jl index 0a6837e67..95031060e 100644 --- a/src/loss/constant/constant.jl +++ b/src/loss/constant/constant.jl @@ -41,9 +41,9 @@ end ### methods ############################################################################################ -objective!(constant::SemConstant, par, model) = constant.c -gradient!(constant::SemConstant, par, model) = zero(par) -hessian!(constant::SemConstant, par, model) = zeros(eltype(par), length(par), length(par)) +objective(constant::SemConstant, model::AbstractSem, par) = constant.c +gradient(constant::SemConstant, model::AbstractSem, par) = zero(par) +hessian(constant::SemConstant, model::AbstractSem, par) = zeros(eltype(par), length(par), length(par)) ############################################################################################ ### Recommended methods diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index 17d0b8c23..aa1c0ee3a 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -43,11 +43,11 @@ end ############################################################################ function SemRidge(; - α_ridge, - which_ridge, + α_ridge, + which_ridge, nparams, - parameter_type = Float64, - imply = nothing, + parameter_type = Float64, + imply = nothing, kwargs...) if eltype(which_ridge) <: Symbol @@ -71,14 +71,14 @@ end ### methods ############################################################################################ -objective!(ridge::SemRidge, par, model) = @views ridge.α*sum(abs2, par[ridge.which]) +objective(ridge::SemRidge, model::AbstractSem, par) = @views ridge.α*sum(abs2, par[ridge.which]) -function gradient!(ridge::SemRidge, par, model) +function gradient(ridge::SemRidge, model::AbstractSem, par) @views ridge.gradient[ridge.which] .= (2*ridge.α)*par[ridge.which] return ridge.gradient end -function hessian!(ridge::SemRidge, par, model) +function hessian(ridge::SemRidge, model::AbstractSem, par) @views @. ridge.hessian[ridge.which_H] .= 2*ridge.α return ridge.hessian end diff --git a/src/objective_gradient_hessian.jl b/src/objective_gradient_hessian.jl index b1119afd2..b064bf075 100644 --- a/src/objective_gradient_hessian.jl +++ b/src/objective_gradient_hessian.jl @@ -1,287 +1,127 @@ -############################################################################################ -# methods for AbstractSem -############################################################################################ - -function objective!(model::AbstractSemSingle, parameters) - objective!(imply(model), parameters, model) - return objective!(loss(model), parameters, model) -end - -function gradient!(gradient, model::AbstractSemSingle, parameters) - fill!(gradient, zero(eltype(gradient))) - gradient!(imply(model), parameters, model) - gradient!(gradient, loss(model), parameters, model) -end - -function hessian!(hessian, model::AbstractSemSingle, parameters) - fill!(hessian, zero(eltype(hessian))) - hessian!(imply(model), parameters, model) - hessian!(hessian, loss(model), parameters, model) -end - -function objective_gradient!(gradient, model::AbstractSemSingle, parameters) - fill!(gradient, zero(eltype(gradient))) - objective_gradient!(imply(model), parameters, model) - objective_gradient!(gradient, loss(model), parameters, model) -end - -function objective_hessian!(hessian, model::AbstractSemSingle, parameters) - fill!(hessian, zero(eltype(hessian))) - objective_hessian!(imply(model), parameters, model) - objective_hessian!(hessian, loss(model), parameters, model) -end - -function gradient_hessian!(gradient, hessian, model::AbstractSemSingle, parameters) - fill!(gradient, zero(eltype(gradient))) - fill!(hessian, zero(eltype(hessian))) - gradient_hessian!(imply(model), parameters, model) - gradient_hessian!(gradient, hessian, loss(model), parameters, model) -end - -function objective_gradient_hessian!(gradient, hessian, model::AbstractSemSingle, parameters) - fill!(gradient, zero(eltype(gradient))) - fill!(hessian, zero(eltype(hessian))) - objective_gradient_hessian!(imply(model), parameters, model) - return objective_gradient_hessian!(gradient, hessian, loss(model), parameters, model) -end +"Specifies whether objective (O), gradient (G) or hessian (H) evaluation is required" +struct EvaluationTargets{O,G,H} end + +EvaluationTargets(objective, gradient, hessian) = + EvaluationTargets{!isnothing(objective),!isnothing(gradient),!isnothing(hessian)}() + +# convenience methods to check type params +is_objective_required(::EvaluationTargets{O}) where O = O +is_gradient_required(::EvaluationTargets{<:Any,G}) where G = G +is_hessian_required(::EvaluationTargets{<:Any,<:Any,H}) where H = H + +# return the tuple of the required results +(::EvaluationTargets{true,false,false})(objective, gradient, hessian) = objective +(::EvaluationTargets{false,true,false})(objective, gradient, hessian) = gradient +(::EvaluationTargets{false,false,true})(objective, gradient, hessian) = hessian +(::EvaluationTargets{true,true,false})(objective, gradient, hessian) = (objective, gradient) +(::EvaluationTargets{true,false,true})(objective, gradient, hessian) = (objective, hessian) +(::EvaluationTargets{false,true,true})(objective, gradient, hessian) = (gradient, hessian) +(::EvaluationTargets{true,true,true})(objective, gradient, hessian) = (objective, gradient, hessian) + +(targets::EvaluationTargets)(arg_tuple::Tuple) = targets(arg_tuple...) + +# dispatch on SemImply +evaluate!(objective, gradient, hessian, loss::SemLossFunction, model::AbstractSem, params) = + evaluate!(objective, gradient, hessian, loss, imply(model), model, params) + +# fallback method +function evaluate!(obj, grad, hess, loss::SemLossFunction, imply::SemImply, model, params) + isnothing(obj) || (obj = objective(loss, imply, model, params)) + isnothing(grad) || copyto!(grad, gradient(loss, imply, model, params)) + isnothing(hess) || copyto!(hess, hessian(loss, imply, model, params)) + return obj +end + +# fallback methods +objective(f::SemLossFunction, imply::SemImply, model, params) = objective(f, model, params) +gradient(f::SemLossFunction, imply::SemImply, model, params) = gradient(f, model, params) +hessian(f::SemLossFunction, imply::SemImply, model, params) = hessian(f, model, params) + +# fallback method for SemImply that calls update_xxx!() methods +function update!(targets::EvaluationTargets, imply::SemImply, model, params) + is_objective_required(targets) && update_objective!(imply, model, params) + is_gradient_required(targets) && update_gradient!(imply, model, params) + is_hessian_required(targets) && update_hessian!(imply, model, params) +end + +# guess objective type +objective_type(model::AbstractSem, params::Any) = Float64 +objective_type(model::AbstractSem, params::AbstractVector{T}) where T <: Number = T +objective_zero(model::AbstractSem, params::Any) = zero(objective_type(model, params)) + +objective_type(objective::T, gradient, hessian) where T <: Number = T +objective_type(objective::Nothing, gradient::AbstractArray{T}, hessian) where T <: Number = T +objective_type(objective::Nothing, gradient::Nothing, hessian::AbstractArray{T}) where T <: Number = T +objective_zero(objective, gradient, hessian) = + zero(objective_type(objective, gradient, hessian)) ############################################################################################ -# methods for SemFiniteDiff +# methods for AbstractSem ############################################################################################ -gradient!(gradient, model::SemFiniteDiff, par) = - FiniteDiff.finite_difference_gradient!(gradient, x -> objective!(model, x), par) - -hessian!(hessian, model::SemFiniteDiff, par) = - FiniteDiff.finite_difference_hessian!(hessian, x -> objective!(model, x), par) - - -function objective_gradient!( - gradient, - model::SemFiniteDiff, - parameters) - gradient!(gradient, model, parameters) - return objective!(model, parameters) -end - -# other methods -function gradient_hessian!( - gradient, - hessian, - model::SemFiniteDiff, - parameters) - gradient!(gradient, model, parameters) - hessian!(hessian, model, parameters) -end - -function objective_hessian!(hessian, model::SemFiniteDiff, parameters) - hessian!(hessian, model, parameters) - return objective!(model, parameters) -end - -function objective_gradient_hessian!( - gradient, - hessian, - model::SemFiniteDiff, - parameters) - hessian!(hessian, model, parameters) - return objective_gradient!(gradient, model, parameters) +function evaluate!(objective, gradient, hessian, model::AbstractSemSingle, params) + targets = EvaluationTargets(objective, gradient, hessian) + # update imply state, its gradient and hessian (if required) + update!(targets, imply(model), model, params) + return evaluate!(!isnothing(objective) ? zero(objective) : nothing, + gradient, hessian, loss(model), model, params) end ############################################################################################ -# methods for SemLoss +# methods for SemFiniteDiff (approximate gradient and hessian with finite differences of objective) ############################################################################################ -function objective!(loss::SemLoss, par, model) - return mapreduce( - (fun, weight) -> weight*objective!(fun, par, model), - +, - loss.functions, loss.weights) -end - -function gradient!(gradient, loss::SemLoss, par, model) - for (lossfun, w) in zip(loss.functions, loss.weights) - new_gradient = gradient!(lossfun, par, model) - gradient .+= w*new_gradient - end -end - -function hessian!(hessian, loss::SemLoss, par, model) - for (lossfun, w) in zip(loss.functions, loss.weights) - hessian .+= w*hessian!(lossfun, par, model) - end -end - -function objective_gradient!(gradient, loss::SemLoss, par, model) - return mapreduce( - (fun, weight) -> objective_gradient_wrap_(gradient, fun, par, model, weight), - +, - loss.functions, loss.weights) -end - -function objective_hessian!(hessian, loss::SemLoss, par, model) - return mapreduce( - (fun, weight) -> objective_hessian_wrap_(hessian, fun, par, model, weight), - +, - loss.functions, loss.weights) -end - -function gradient_hessian!(gradient, hessian, loss::SemLoss, par, model) - for (lossfun, w) in zip(loss.functions, loss.weights) - new_gradient, new_hessian = gradient_hessian!(lossfun, par, model) - gradient .+= w*new_gradient - hessian .+= w*new_hessian +function evaluate!(objective, gradient, hessian, model::SemFiniteDiff, params) + function obj(p) + # recalculate imply state for p + update!(EvaluationTargets{true,false,false}(), imply(model), model, p) + evaluate!(objective_zero(objective, gradient, hessian), + nothing, nothing, loss(model), model, p) end + isnothing(gradient) || FiniteDiff.finite_difference_gradient!(gradient, obj, params) + isnothing(hessian) || FiniteDiff.finite_difference_hessian!(hessian, obj, params) + return !isnothing(objective) ? obj(params) : nothing end -function objective_gradient_hessian!(gradient, hessian, loss::SemLoss, par, model) - return mapreduce( - (fun, weight) -> objective_gradient_hessian_wrap_(gradient, hessian, fun, par, model, weight), - +, - loss.functions, loss.weights) -end - -# wrapper to update gradient/hessian and return objective value -function objective_gradient_wrap_(gradient, lossfun, par, model, w) - new_objective, new_gradient = objective_gradient!(lossfun, par, model) - gradient .+= w*new_gradient - return w*new_objective -end - -function objective_hessian_wrap_(hessian, lossfun, par, model, w) - new_objective, new_hessian = objective_hessian!(lossfun, par, model) - hessian .+= w*new_hessian - return w*new_objective -end - -function objective_gradient_hessian_wrap_(gradient, hessian, lossfun, par, model, w) - new_objective, new_gradient, new_hessian = objective_gradient_hessian!(lossfun, par, model) - gradient .+= w*new_gradient - hessian .+= w*new_hessian - return w*new_objective -end +objective(model::AbstractSem, params) = + evaluate!(objective_zero(model, params), nothing, nothing, model, params) ############################################################################################ -# methods for SemEnsemble +# methods for SemLoss (weighted sum of individual SemLossFunctions) ############################################################################################ -function objective!(ensemble::SemEnsemble, par) - return mapreduce( - (model, weight) -> weight*objective!(model, par), - +, - ensemble.sems, ensemble.weights) -end - -function gradient!(gradient, ensemble::SemEnsemble, par) - fill!(gradient, zero(eltype(gradient))) - for (model, w) in zip(ensemble.sems, ensemble.weights) - gradient_new = similar(gradient) - gradient!(gradient_new, model, par) - gradient .+= w*gradient_new - end -end - -function hessian!(hessian, ensemble::SemEnsemble, par) - fill!(hessian, zero(eltype(hessian))) - for (model, w) in zip(ensemble.sems, ensemble.weights) - hessian_new = similar(hessian) - hessian!(hessian_new, model, par) - hessian .+= w*hessian_new - end -end - -function objective_gradient!(gradient, ensemble::SemEnsemble, par) - fill!(gradient, zero(eltype(gradient))) - return mapreduce( - (model, weight) -> objective_gradient_wrap_(gradient, model, par, weight), - +, - ensemble.sems, ensemble.weights) -end - -function objective_hessian!(hessian, ensemble::SemEnsemble, par) - fill!(hessian, zero(eltype(hessian))) - return mapreduce( - (model, weight) -> objective_hessian_wrap_(hessian, model, par, weight), - +, - ensemble.sems, ensemble.weights) -end - -function gradient_hessian!(gradient, hessian, ensemble::SemEnsemble, par) - fill!(gradient, zero(eltype(gradient))) - fill!(hessian, zero(eltype(hessian))) - for (model, w) in zip(ensemble.sems, ensemble.weights) - - new_gradient = similar(gradient) - new_hessian = similar(hessian) - - gradient_hessian!(new_gradient, new_hessian, model, par) - - gradient .+= w*new_gradient - hessian .+= w*new_hessian - +function evaluate!(objective, gradient, hessian, loss::SemLoss, model::AbstractSem, params) + isnothing(objective) || (objective = zero(objective)) + isnothing(gradient) || fill!(gradient, zero(eltype(gradient))) + isnothing(hessian) || fill!(hessian, zero(eltype(hessian))) + f_grad = isnothing(gradient) ? nothing : similar(gradient) + f_hess = isnothing(hessian) ? nothing : similar(hessian) + for (f, weight) in zip(loss.functions, loss.weights) + f_obj = evaluate!(objective, f_grad, f_hess, f, model, params) + isnothing(objective) || (objective += weight*f_obj) + isnothing(gradient) || (gradient .+= weight*f_grad) + isnothing(hessian) || (hessian .+= weight*f_hess) end -end - -function objective_gradient_hessian!(gradient, hessian, ensemble::SemEnsemble, par) - fill!(gradient, zero(eltype(gradient))) - fill!(hessian, zero(eltype(hessian))) - return mapreduce( - (model, weight) -> objective_gradient_hessian_wrap_(gradient, hessian, model, par, model, weight), - +, - ensemble.sems, ensemble.weights) -end - -# wrapper to update gradient/hessian and return objective value -function objective_gradient_wrap_(gradient, model::AbstractSemSingle, par, w) - gradient_pre = similar(gradient) - new_objective = objective_gradient!(gradient_pre, model, par) - gradient .+= w*gradient_pre - return w*new_objective -end - -function objective_hessian_wrap_(hessian, model::AbstractSemSingle, par, w) - hessian_pre = similar(hessian) - new_objective = objective_hessian!(hessian_pre, model, par) - hessian .+= w*new_hessian - return w*new_objective -end - -function objective_gradient_hessian_wrap_(gradient, hessian, model::AbstractSemSingle, par, w) - gradient_pre = similar(gradient) - hessian_pre = similar(hessian) - new_objective = objective_gradient_hessian!(gradient_pre, hessian_pre, model, par) - gradient .+= w*new_gradient - hessian .+= w*new_hessian - return w*new_objective + return objective end ############################################################################################ -# generic methods for loss functions +# methods for SemEnsemble (weighted sum of individual AbstractSemSingle models) ############################################################################################ -function objective_gradient!(lossfun::SemLossFunction, par, model) - objective = objective!(lossfun::SemLossFunction, par, model) - gradient = gradient!(lossfun::SemLossFunction, par, model) - return objective, gradient -end - -function objective_hessian!(lossfun::SemLossFunction, par, model) - objective = objective!(lossfun::SemLossFunction, par, model) - hessian = hessian!(lossfun::SemLossFunction, par, model) - return objective, hessian -end - -function gradient_hessian!(lossfun::SemLossFunction, par, model) - gradient = gradient!(lossfun::SemLossFunction, par, model) - hessian = hessian!(lossfun::SemLossFunction, par, model) - return gradient, hessian -end - -function objective_gradient_hessian!(lossfun::SemLossFunction, par, model) - objective = objective!(lossfun::SemLossFunction, par, model) - gradient = gradient!(lossfun::SemLossFunction, par, model) - hessian = hessian!(lossfun::SemLossFunction, par, model) - return objective, gradient, hessian +function evaluate!(objective, gradient, hessian, ensemble::SemEnsemble, params) + isnothing(objective) || (objective = zero(objective)) + isnothing(gradient) || fill!(gradient, zero(eltype(gradient))) + isnothing(hessian) || fill!(hessian, zero(eltype(hessian))) + sem_grad = isnothing(gradient) ? nothing : similar(gradient) + sem_hess = isnothing(hessian) ? nothing : similar(hessian) + for (sem, weight) in zip(ensemble.sems, ensemble.weights) + sem_obj = evaluate!(objective, sem_grad, sem_hess, sem, params) + isnothing(objective) || (objective += weight*sem_obj) + isnothing(gradient) || (gradient .+= weight*sem_grad) + isnothing(hessian) || (hessian .+= weight*sem_hess) + end + return objective end # throw an error by default if gradient! and hessian! are not implemented @@ -292,35 +132,6 @@ end hessian!(lossfun::SemLossFunction, par, model) = throw(ArgumentError("hessian for $(typeof(lossfun).name.wrapper) is not available")) =# -############################################################################################ -# generic methods for imply -############################################################################################ - -function objective_gradient!(semimp::SemImply, par, model) - objective!(semimp::SemImply, par, model) - gradient!(semimp::SemImply, par, model) - return nothing -end - -function objective_hessian!(semimp::SemImply, par, model) - objective!(semimp::SemImply, par, model) - hessian!(semimp::SemImply, par, model) - return nothing -end - -function gradient_hessian!(semimp::SemImply, par, model) - gradient!(semimp::SemImply, par, model) - hessian!(semimp::SemImply, par, model) - return nothing -end - -function objective_gradient_hessian!(semimp::SemImply, par, model) - objective!(semimp::SemImply, par, model) - gradient!(semimp::SemImply, par, model) - hessian!(semimp::SemImply, par, model) - return nothing -end - ############################################################################################ # Documentation ############################################################################################ @@ -365,4 +176,19 @@ To implement a new `SemImply` or `SemLossFunction` type, you can add a method fo To implement a new `AbstractSem` subtype, you can add a method for hessian!(hessian, model::MyNewType, parameters) """ -function hessian! end \ No newline at end of file +function hessian! end + +objective!(model::AbstractSem, params) = + evaluate!(objective_zero(model, params), nothing, nothing, model, params) +gradient!(gradient, model::AbstractSem, params) = + evaluate!(nothing, gradient, nothing, model, params) +hessian!(hessian, model::AbstractSem, params) = + evaluate!(nothing, nothing, hessian, model, params) +objective_gradient!(gradient, model::AbstractSem, params) = + evaluate!(objective_zero(model, params), gradient, nothing, model, params) +objective_hessian!(hessian, model::AbstractSem, params) = + evaluate!(objective_zero(model, params), nothing, hessian, model, params) +gradient_hessian!(gradient, hessian, model::AbstractSem, params) = + evaluate!(nothing, gradient, hessian, model, params) +objective_gradient_hessian!(gradient, hessian, model::AbstractSem, params) = + evaluate!(objective_zero(model, params), gradient, hessian, model, params) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 41d7d55bb..c0d117030 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -79,7 +79,7 @@ end grad = similar(start_test) gradient!(grad, model_ml_multigroup, rand(36)) -grad_fd = FiniteDiff.finite_difference_gradient(x -> objective!(model_ml_multigroup, x), start_test) +grad_fd = FiniteDiff.finite_difference_gradient(x -> SEM.objective!(model_ml_multigroup, x), start_test) # fit @testset "ml_solution_multigroup | sorted" begin @@ -114,22 +114,19 @@ end # ML estimation - user defined loss function ############################################################################################ -struct UserSemML <: SemLossFunction{ExactHessian} end +import LinearAlgebra: isposdef, logdet, tr, inv -############################################################################################ -### functors -############################################################################################ +SEM = StructuralEquationModels -import LinearAlgebra: isposdef, logdet, tr, inv -import StructuralEquationModels: Σ, obs_cov, objective! - -function objective!(semml::UserSemML, parameters, model::AbstractSem) - let Σ = Σ(imply(model)), Σₒ = obs_cov(observed(model)) - if !isposdef(Σ) - return Inf - else - return logdet(Σ) + tr(inv(Σ)*Σₒ) - end +struct UserSemML <: SemLossFunction{ExactHessian} end + +function SEM.objective(ml::UserSemML, model::AbstractSem, params) + Σ = imply(model).Σ + Σₒ = SEM.obs_cov(observed(model)) + if !isposdef(Σ) + return Inf + else + return logdet(Σ) + tr(inv(Σ)*Σₒ) end end From c69f2629bd792e41616ca21f20356bf26f5a556d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:46:53 -0700 Subject: [PATCH 061/174] use ternary op as intended --- src/imply/RAM/generic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index f2a3ca48b..0f71fc554 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -115,7 +115,7 @@ function RAM(; # get indices A_indices = copy(ram_matrices.A_ind) S_indices = copy(ram_matrices.S_ind) - !isnothing(ram_matrices.M_ind) ? M_indices = copy(ram_matrices.M_ind) : M_indices = nothing + M_indices = !isnothing(ram_matrices.M_ind) ? copy(ram_matrices.M_ind) : nothing #preallocate arrays A_pre = zeros(n_var, n_var) From 2133f974d10862ffe1f493b0b5e5d01c9a10561b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:48:22 -0700 Subject: [PATCH 062/174] symbolic: constrain to tril before simplifying --- src/imply/RAM/symbolic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 5328cc9b8..e01f50f4b 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -239,10 +239,10 @@ function get_Σ_symbolic_RAM(S, A, F; vech = false) Σ_symbolic = F*invia*S*permutedims(invia)*permutedims(F) Σ_symbolic = Array(Σ_symbolic) # Σ_symbolic = Symbolics.simplify.(Σ_symbolic) + if vech Σ_symbolic = Σ_symbolic[tril(trues(size(F, 1), size(F, 1)))] end Threads.@threads for i in eachindex(Σ_symbolic) Σ_symbolic[i] = Symbolics.simplify(Σ_symbolic[i]) end - if vech Σ_symbolic = Σ_symbolic[tril(trues(size(F, 1), size(F, 1)))] end return Σ_symbolic end From a1a6f10544288777a8c813a0ac36fca142ead586 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 13:51:41 -0700 Subject: [PATCH 063/174] fix dangling whitespace --- src/additional_functions/helper.jl | 10 ++--- src/additional_functions/parameters.jl | 4 -- .../start_val/start_fabin3.jl | 26 ++++++------ src/frontend/fit/standard_errors/hessian.jl | 6 +-- .../specification/EnsembleParameterTable.jl | 2 +- src/frontend/specification/ParameterTable.jl | 22 +++++----- src/frontend/specification/Sem.jl | 2 +- src/imply/RAM/generic.jl | 4 +- src/imply/RAM/symbolic.jl | 2 +- src/loss/WLS/WLS.jl | 17 ++++---- src/types.jl | 6 +-- test/unit_tests/data_input_formats.jl | 42 +++++++++---------- 12 files changed, 70 insertions(+), 73 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index d7da48c25..99bcfb3a8 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -10,7 +10,7 @@ function neumann_series(mat::SparseMatrixCSC) return inverse end -#= +#= function make_onelement_array(A) isa(A, Array) ? nothing : (A = [A]) return A @@ -129,7 +129,7 @@ function duplication_matrix(nobs) u[Int((j-1)*nobs + i-0.5*j*(j-1))] = 1 T = zeros(nobs, nobs) T[j,i] = 1; T[i, j] = 1 - Dt += u*transpose(vec(T)) + Dt += u*transpose(vec(T)) end end D = transpose(Dt) @@ -148,7 +148,7 @@ function elimination_matrix(nobs) u[Int((j-1)*nobs + i-0.5*j*(j-1))] = 1 T = zeros(nobs, nobs) T[i, j] = 1 - L += u*transpose(vec(T)) + L += u*transpose(vec(T)) end end return L @@ -215,11 +215,11 @@ function get_commutation_lookup(n2::Int64) end return lookup - + end function commutation_matrix_pre_square!(A::SparseMatrixCSC, lookup) # comuptes B + KₙA - + for (i, rowind) in enumerate(A.rowval) A.rowval[i] = lookup[rowind] end diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl index 731116f7c..67fb37c72 100644 --- a/src/additional_functions/parameters.jl +++ b/src/additional_functions/parameters.jl @@ -16,18 +16,14 @@ function fill_A_S_M!( for index_S in iS S[index_S] = par end - end if !isnothing(M) - @inbounds for (iM, par) in zip(M_indices, parameters) for index_M in iM M[index_M] = par end - end - end end diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 5124a682d..9a4a706b5 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -1,6 +1,6 @@ """ start_fabin3(model) - + Return a vector of FABIN 3 starting values (see Hägglund 1982). Not available for ensemble models. """ @@ -8,20 +8,20 @@ function start_fabin3 end # splice model and loss functions function start_fabin3( - model::AbstractSemSingle; + model::AbstractSemSingle; kwargs...) return start_fabin3( - model.observed, + model.observed, model.imply, - model.optimizer, + model.optimizer, model.loss.functions..., kwargs...) end function start_fabin3( - observed, - imply, - optimizer, + observed, + imply, + optimizer, args...; kwargs...) return start_fabin3( @@ -32,10 +32,10 @@ end # SemObservedMissing function start_fabin3( - observed::SemObservedMissing, - imply, - optimizer, - args...; + observed::SemObservedMissing, + imply, + optimizer, + args...; kwargs...) if !observed.em_model.fitted @@ -77,8 +77,8 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) if !isnothing(M) in_M = length.(M_ind) .!= 0 in_any = in_A .| in_S .| in_M - else - in_any = in_A .| in_S + else + in_any = in_A .| in_S end if !all(in_any) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index f713920d7..0607a8895 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -4,9 +4,9 @@ Return hessian based standard errors. # Arguments -- `hessian`: how to compute the hessian. Options are +- `hessian`: how to compute the hessian. Options are - `:analytic`: (only if an analytic hessian for the model can be computed) - - `:finitediff`: for finite difference approximation + - `:finitediff`: for finite difference approximation """ function se_hessian(sem_fit::SemFit; hessian = :finitediff) @@ -36,7 +36,7 @@ function se_hessian(sem_fit::SemFit; hessian = :finitediff) end # Addition functions ------------------------------------------------------------- -H_scaling(model::AbstractSemSingle) = +H_scaling(model::AbstractSemSingle) = H_scaling( model, model.observed, diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 8798a569a..df84441ac 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -37,7 +37,7 @@ function Base.convert(::Type{Dict{K, RAMMatrices}}, end #= function DataFrame( - partable::ParameterTable; + partable::ParameterTable; columns = nothing) if isnothing(columns) columns = keys(partable.columns) end out = DataFrame([key => partable.columns[key] for key in columns]) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 74114942f..b2d966f69 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -224,9 +224,9 @@ update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, # update estimates ------------------------------------------------------------------------- """ update_estimate!( - partable::AbstractParameterTable, + partable::AbstractParameterTable, sem_fit::SemFit) - + Write parameter estimates from `sem_fit` to the `:estimate` column of `partable` """ update_estimate!(partable::AbstractParameterTable, sem_fit::SemFit) = @@ -236,7 +236,7 @@ update_estimate!(partable::AbstractParameterTable, sem_fit::SemFit) = """ update_start!(partable::AbstractParameterTable, sem_fit::SemFit) update_start!(partable::AbstractParameterTable, model::AbstractSem, start_val; kwargs...) - + Write starting values from `sem_fit` or `start_val` to the `:estimate` column of `partable`. # Arguments @@ -248,9 +248,9 @@ update_start!(partable::AbstractParameterTable, sem_fit::SemFit) = update_partable!(partable, sem_fit, sem_fit.start_val, :start) function update_start!( - partable::AbstractParameterTable, - model::AbstractSem, - start_val; + partable::AbstractParameterTable, + model::AbstractSem, + start_val; kwargs...) if !(start_val isa Vector) start_val = start_val(model; kwargs...) @@ -261,10 +261,10 @@ end # update partable standard errors ---------------------------------------------------------- """ update_se_hessian!( - partable::AbstractParameterTable, - sem_fit::SemFit; + partable::AbstractParameterTable, + sem_fit::SemFit; hessian = :finitediff) - + Write hessian standard errors computed for `sem_fit` to the `:se` column of `partable` # Arguments @@ -274,8 +274,8 @@ Write hessian standard errors computed for `sem_fit` to the `:se` column of `par """ function update_se_hessian!( - partable::AbstractParameterTable, - sem_fit::SemFit; + partable::AbstractParameterTable, + sem_fit::SemFit; hessian = :finitediff) se = se_hessian(sem_fit; hessian = hessian) return update_partable!(partable, sem_fit, se, :se) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 96f07f671..0c22db1d8 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -139,7 +139,7 @@ function Base.show(io::IO, sem::SemFiniteDiff{O, I, L, D}) where {O, I, L, D} print(io, "- Fields \n") print(io, " observed: $(nameof(O)) \n") print(io, " imply: $(nameof(I)) \n") - print(io, " optimizer: $(nameof(D)) \n") + print(io, " optimizer: $(nameof(D)) \n") end function Base.show(io::IO, loss::SemLoss) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 0f71fc554..02dcdaad4 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -123,7 +123,7 @@ function RAM(; M_pre = !isnothing(M_indices) ? zeros(n_var) : nothing set_RAMConstants!(A_pre, S_pre, M_pre, ram_matrices.constants) - + A_pre = check_acyclic(A_pre, n_par, A_indices) # pre-allocate some matrices @@ -225,7 +225,7 @@ end identifier(imply::RAM) = imply.identifier nparams(imply::RAM) = nparams(imply.ram_matrices) -function update_observed(imply::RAM, observed::SemObserved; kwargs...) +function update_observed(imply::RAM, observed::SemObserved; kwargs...) if n_man(observed) == size(imply.Σ, 1) return imply else diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index e01f50f4b..3eed25a8a 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -147,7 +147,7 @@ function RAMSymbolic(; for i in 1:n_sig ∇²Σ_symbolic += J[i]*∇²Σ_symbolic_vec[i] end - + ∇²Σ_function = Symbolics.build_function(∇²Σ_symbolic, J, par, expression=Val{false})[2] ∇²Σ = zeros(n_par, n_par) else diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 45891a226..bb9647209 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -10,20 +10,20 @@ Weighted least squares estimation. SemWLS(; observed, - meanstructure = false, - wls_weight_matrix = nothing, - wls_weight_matrix_mean = nothing, - approximate_hessian = false, + meanstructure = false, + wls_weight_matrix = nothing, + wls_weight_matrix_mean = nothing, + approximate_hessian = false, kwargs...) # Arguments - `observed`: the `SemObserved` part of the model - `meanstructure::Bool`: does the model have a meanstructure? - `approximate_hessian::Bool`: should the hessian be swapped for an approximation -- `wls_weight_matrix`: the weight matrix for weighted least squares. - Defaults to GLS estimation (``0.5*(D^T*kron(S,S)*D)`` where D is the duplication matrix +- `wls_weight_matrix`: the weight matrix for weighted least squares. + Defaults to GLS estimation (``0.5*(D^T*kron(S,S)*D)`` where D is the duplication matrix and S is the inverse ob the observed covariance matrix) -- `wls_weight_matrix_mean`: the weight matrix for the mean part of weighted least squares. +- `wls_weight_matrix_mean`: the weight matrix for the mean part of weighted least squares. Defaults to GLS estimation (the inverse of the observed covariance matrix) # Examples @@ -135,4 +135,5 @@ end ### Recommended methods ############################################################################################ -update_observed(lossfun::SemWLS, observed::SemObserved; kwargs...) = SemWLS(;observed = observed, kwargs...) \ No newline at end of file +update_observed(lossfun::SemWLS, observed::SemObserved; kwargs...) = + SemWLS(; observed = observed, kwargs...) \ No newline at end of file diff --git a/src/types.jl b/src/types.jl index 4b671e5ed..169936e52 100644 --- a/src/types.jl +++ b/src/types.jl @@ -59,7 +59,7 @@ end function SemLoss(functions...; loss_weights = nothing, kwargs...) - if !isnothing(loss_weights) + if !isnothing(loss_weights) loss_weights = SemWeight.(loss_weights) else loss_weights = Tuple(SemWeight(nothing) for _ in 1:length(functions)) @@ -174,7 +174,7 @@ Constructor for ensemble models. - `weights::Vector`: Weights for each model. Defaults to the number of observed data points. All additional kwargs are passed down to the constructor for the optimizer field. - + Returns a SemEnsemble with fields - `n::Int`: Number of models. - `sems::Tuple`: `AbstractSem`s. @@ -194,7 +194,7 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing n = length(models) # default weights - + if isnothing(weights) nobs_total = sum(n_obs.(models)) weights = [n_obs(model)/nobs_total for model in models] diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 992140f57..767408e4c 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -57,16 +57,16 @@ observed_nospec = SemObservedData( ) observed_matrix = SemObservedData( - specification = spec, - data = dat_matrix, + specification = spec, + data = dat_matrix, obs_colnames = Symbol.(names(dat)) ) -all_equal_cov = +all_equal_cov = (obs_cov(observed) == obs_cov(observed_nospec)) & (obs_cov(observed) == obs_cov(observed_matrix)) -all_equal_data = +all_equal_data = (get_data(observed) == get_data(observed_nospec)) & (get_data(observed) == get_data(observed_matrix)) @@ -90,16 +90,16 @@ observed_shuffle = SemObservedData( ) observed_matrix_shuffle = SemObservedData( - specification = spec, - data = shuffle_dat_matrix, + specification = spec, + data = shuffle_dat_matrix, obs_colnames = shuffle_names ) -all_equal_cov_suffled = +all_equal_cov_suffled = (obs_cov(observed) == obs_cov(observed_shuffle)) & (obs_cov(observed) == obs_cov(observed_matrix_shuffle)) -all_equal_data_suffled = +all_equal_data_suffled = (get_data(observed) == get_data(observed_shuffle)) & (get_data(observed) == get_data(observed_matrix_shuffle)) @@ -146,13 +146,13 @@ observed_nospec = SemObservedData( ) observed_matrix = SemObservedData( - specification = spec, - data = dat_matrix, + specification = spec, + data = dat_matrix, obs_colnames = Symbol.(names(dat)), meanstructure = true ) -all_equal_mean = +all_equal_mean = (obs_mean(observed) == obs_mean(observed_nospec)) & (obs_mean(observed) == obs_mean(observed_matrix)) @@ -176,15 +176,15 @@ observed_shuffle = SemObservedData( ) observed_matrix_shuffle = SemObservedData( - specification = spec, - data = shuffle_dat_matrix, + specification = spec, + data = shuffle_dat_matrix, obs_colnames = shuffle_names, meanstructure = true ) -all_equal_mean_suffled = +all_equal_mean_suffled = (obs_mean(observed) == obs_mean(observed_shuffle)) & - (obs_mean(observed) == obs_mean(observed_matrix_shuffle)) + (obs_mean(observed) == obs_mean(observed_matrix_shuffle)) @testset "unit tests | SemObservedData | input formats shuffled - mean" begin @@ -367,12 +367,12 @@ observed_nospec = SemObservedMissing( ) observed_matrix = SemObservedMissing( - specification = spec, - data = dat_missing_matrix, + specification = spec, + data = dat_missing_matrix, obs_colnames = Symbol.(names(dat)) ) -all_equal_data = +all_equal_data = isequal(get_data(observed), get_data(observed_nospec)) & isequal(get_data(observed), get_data(observed_matrix)) @@ -395,12 +395,12 @@ observed_shuffle = SemObservedMissing( ) observed_matrix_shuffle = SemObservedMissing( - specification = spec, - data = shuffle_dat_missing_matrix, + specification = spec, + data = shuffle_dat_missing_matrix, obs_colnames = shuffle_names ) -all_equal_data_shuffled = +all_equal_data_shuffled = isequal(get_data(observed), get_data(observed_shuffle)) & isequal(get_data(observed), get_data(observed_matrix_shuffle)) From b2eb026994b6643e65071f67dc5e67c879b4ac30 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 21:43:05 -0700 Subject: [PATCH 064/174] check_acyclic: notify if matrix is triangular --- src/imply/RAM/generic.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 02dcdaad4..5432d86d5 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -252,12 +252,13 @@ function check_acyclic(A_pre, n_par, A_indices) # check if A is lower or upper triangular if istril(A_rand) - A_pre = LowerTriangular(A_pre) + @info "A matrix is lower triangular" + return LowerTriangular(A_pre) elseif istriu(A_rand) - A_pre = UpperTriangular(A_pre) + @info "A matrix is upper triangular" + return UpperTriangular(A_pre) elseif acyclic @info "Your model is acyclic, specifying the A Matrix as either Upper or Lower Triangular can have great performance benefits.\n" maxlog=1 + return A_pre end - - return A_pre end \ No newline at end of file From e61daf25a36c514d769a2aeb4eed6e75477539b4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 15 Mar 2024 08:36:18 -0700 Subject: [PATCH 065/174] tests/examples: import -> using no declarations, so import is not required --- test/examples/multigroup/multigroup.jl | 4 ++-- test/examples/political_democracy/constructor.jl | 2 +- test/examples/political_democracy/political_democracy.jl | 2 +- .../examples/recover_parameters/recover_parameters_twofact.jl | 4 ++-- test/unit_tests/data_input_formats.jl | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 5a34c4d86..615f3d6ed 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -1,6 +1,6 @@ using StructuralEquationModels, Test, FiniteDiff -import LinearAlgebra: diagind, LowerTriangular -# import StructuralEquationModels as SEM +using LinearAlgebra: diagind, LowerTriangular +# using StructuralEquationModels as SEM include( joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), "test/examples/helper.jl") diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 03c5b05ec..3e6256181 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -1,4 +1,4 @@ -import Statistics: cov, mean +using Statistics: cov, mean ############################################################################################ ### models w.o. meanstructure diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 6e7162173..748d232e1 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,5 +1,5 @@ using StructuralEquationModels, Test, FiniteDiff -# import StructuralEquationModels as SEM + include( joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), "test/examples/helper.jl") diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index f0b099f08..7cadbee0c 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -1,5 +1,5 @@ using StructuralEquationModels, Distributions, Random, Optim, LineSearches -import StructuralEquationModels as SEM + include( joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), "test/examples/helper.jl") @@ -51,7 +51,7 @@ Random.seed!(1234) x = transpose(rand(true_dist, 100000)) semobserved = SemObservedData(data = x, specification = nothing) -loss_ml = SemLoss(SEM.SemML(;observed = semobserved, nparams = length(start))) +loss_ml = SemLoss(SemML(;observed = semobserved, nparams = length(start))) optimizer = SemOptimizerOptim( diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 767408e4c..a83bb2527 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -1,5 +1,5 @@ using StructuralEquationModels, Test, Statistics -import StructuralEquationModels: obs_cov, obs_mean, get_data +using StructuralEquationModels: obs_cov, obs_mean, get_data # unexported ### model specification -------------------------------------------------------------------- From 6cc0446b5cd870922b4ae40db463667f275bf933 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 21:47:17 -0700 Subject: [PATCH 066/174] fix dangling spaces --- test/examples/multigroup/build_models.jl | 40 +++++++++---------- test/examples/multigroup/multigroup.jl | 16 ++++---- .../political_democracy/constructor.jl | 26 ++++++------ .../political_democracy.jl | 12 +++--- .../recover_parameters_twofact.jl | 8 ++-- 5 files changed, 51 insertions(+), 51 deletions(-) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index c0d117030..36e887486 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -27,7 +27,7 @@ end solution = sem_fit(model_ml_multigroup) update_estimate!(partable, solution) @test compare_estimates( - partable, + partable, solution_lav[:parameter_estimates_ml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) end @@ -35,14 +35,14 @@ end @testset "fitmeasures/se_ml" begin solution_ml = sem_fit(model_ml_multigroup) @test all(test_fitmeasures( - fit_measures(solution_ml), + fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7)) update_partable!( partable, identifier(model_ml_multigroup), se_hessian(solution_ml), :se) @test compare_estimates( - partable, - solution_lav[:parameter_estimates_ml]; atol = 1e-3, + partable, + solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) end @@ -86,7 +86,7 @@ grad_fd = FiniteDiff.finite_difference_gradient(x -> SEM.objective!(model_ml_mul solution = sem_fit(model_ml_multigroup) update_estimate!(partable_s, solution) @test compare_estimates( - partable, + partable, solution_lav[:parameter_estimates_ml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) end @@ -94,14 +94,14 @@ end @testset "fitmeasures/se_ml | sorted" begin solution_ml = sem_fit(model_ml_multigroup) @test all(test_fitmeasures( - fit_measures(solution_ml), + fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7)) update_partable!( partable_s, identifier(model_ml_multigroup), se_hessian(solution_ml), :se) @test compare_estimates( - partable_s, - solution_lav[:parameter_estimates_ml]; atol = 1e-3, + partable_s, + solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) end @@ -114,7 +114,7 @@ end # ML estimation - user defined loss function ############################################################################################ -import LinearAlgebra: isposdef, logdet, tr, inv +using LinearAlgebra: isposdef, logdet, tr, inv SEM = StructuralEquationModels @@ -155,7 +155,7 @@ end solution = sem_fit(model_ml_multigroup) update_estimate!(partable, solution) @test compare_estimates( - partable, + partable, solution_lav[:parameter_estimates_ml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) end @@ -188,7 +188,7 @@ end solution = sem_fit(model_ls_multigroup) update_estimate!(partable, solution) @test compare_estimates( - partable, + partable, solution_lav[:parameter_estimates_ls]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) end @@ -196,14 +196,14 @@ end @testset "fitmeasures/se_ls" begin solution_ls = sem_fit(model_ls_multigroup) @test all(test_fitmeasures( - fit_measures(solution_ls), + fit_measures(solution_ls), solution_lav[:fitmeasures_ls]; fitmeasure_names = fitmeasure_names_ls, rtol = 1e-2, atol = 1e-5)) update_partable!( partable, identifier(model_ls_multigroup), se_hessian(solution_ls), :se) @test compare_estimates( - partable, + partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, col = :se, lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) @@ -242,11 +242,11 @@ model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) ############################################################################################ start_test = [ - fill(0.5, 6); - fill(1.0, 9); - 0.05; 0.01; 0.01; 0.05; 0.01; 0.05; + fill(0.5, 6); + fill(1.0, 9); + 0.05; 0.01; 0.01; 0.05; 0.01; 0.05; fill(0.01, 9); - fill(1.0, 9); + fill(1.0, 9); 0.05; 0.01; 0.01; 0.05; 0.01; 0.05; fill(0.01, 9)] @@ -259,7 +259,7 @@ end solution = sem_fit(model_ml_multigroup) update_estimate!(partable_miss, solution) @test compare_estimates( - partable_miss, + partable_miss, solution_lav[:parameter_estimates_fiml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) end @@ -267,13 +267,13 @@ end @testset "fitmeasures/se_fiml" begin solution = sem_fit(model_ml_multigroup) @test all(test_fitmeasures( - fit_measures(solution), + fit_measures(solution), solution_lav[:fitmeasures_fiml]; rtol = 1e-3, atol = 0)) update_partable!( partable_miss, identifier(model_ml_multigroup), se_hessian(solution), :se) @test compare_estimates( - partable_miss, + partable_miss, solution_lav[:parameter_estimates_fiml]; atol = 1e-3, col = :se, lav_col = :se, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 615f3d6ed..60345dace 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -2,7 +2,7 @@ using StructuralEquationModels, Test, FiniteDiff using LinearAlgebra: diagind, LowerTriangular # using StructuralEquationModels as SEM include( - joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), + joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), "test/examples/helper.jl") ) @@ -65,7 +65,7 @@ partable = EnsembleParameterTable( specification_miss_g1 = nothing specification_miss_g2 = nothing -start_test = [fill(1.0, 9); fill(0.05, 3); fill(0.01, 3); fill(0.5, 6); fill(1.0, 9); +start_test = [fill(1.0, 9); fill(0.05, 3); fill(0.01, 3); fill(0.5, 6); fill(1.0, 9); fill(0.05, 3); fill(0.01, 3)] semoptimizer = SemOptimizerOptim @@ -91,7 +91,7 @@ graph = @StenoGraph begin end partable = EnsembleParameterTable(; - graph = graph, + graph = graph, observed_vars = observed_vars, latent_vars = latent_vars, groups = [:Pasteur, :Grant_White]) @@ -119,7 +119,7 @@ graph = @StenoGraph begin end partable_miss = EnsembleParameterTable(; - graph = graph, + graph = graph, observed_vars = observed_vars, latent_vars = latent_vars, groups = [:Pasteur, :Grant_White]) @@ -130,11 +130,11 @@ specification_miss_g1 = specification_miss[:Pasteur] specification_miss_g2 = specification_miss[:Grant_White] start_test = [ - fill(0.5, 6); - fill(1.0, 9); 0.05; 0.01; 0.01; 0.05; 0.01; 0.05; + fill(0.5, 6); + fill(1.0, 9); 0.05; 0.01; 0.01; 0.05; 0.01; 0.05; fill(1.0, 9); 0.05; 0.01; 0.01; 0.05; 0.01; 0.05] semoptimizer = SemOptimizerOptim -@testset "Graph → Partable → RAMMatrices | constructor | Optim" begin - include("build_models.jl") +@testset "Graph → Partable → RAMMatrices | constructor | Optim" begin + include("build_models.jl") end \ No newline at end of file diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 3e6256181..dd10f4be3 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -113,7 +113,7 @@ end solution_ml = sem_fit(model_ml) solution_ml_weighted = sem_fit(model_ml_weighted) @test isapprox(solution(solution_ml), solution(solution_ml_weighted), rtol = 1e-3) - @test isapprox(n_obs(model_ml)*StructuralEquationModels.minimum(solution_ml), + @test isapprox(n_obs(model_ml)*StructuralEquationModels.minimum(solution_ml), StructuralEquationModels.minimum(solution_ml_weighted), rtol = 1e-6) end @@ -123,23 +123,23 @@ end @testset "fitmeasures/se_ml" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; + @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3)) update_partable!(partable, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ml]; + @test compare_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se) end @testset "fitmeasures/se_ls" begin solution_ls = sem_fit(model_ls_sym) fm = fit_measures(solution_ls) - @test all(test_fitmeasures(fm, solution_lav[:fitmeasures_ls]; atol = 1e-3, + @test all(test_fitmeasures(fm, solution_lav[:fitmeasures_ls]; atol = 1e-3, fitmeasure_names = fitmeasure_names_ls)) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable, identifier(model_ls_sym), se_hessian(solution_ls), :se) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, + @test compare_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, col = :se, lav_col = :se) end @@ -157,7 +157,7 @@ if semoptimizer == SemOptimizerOptim loss = SemWLS, hessian = true, algorithm = Newton( - ;linesearch = BackTracking(order=3), + ;linesearch = BackTracking(order=3), alphaguess = InitialHagerZhang()) ) @@ -271,20 +271,20 @@ end @testset "fitmeasures/se_ml_mean" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; + @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; atol = 0.002)) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; + @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; atol = 0.002, col = :se, lav_col = :se) end @testset "fitmeasures/se_ls_mean" begin solution_ls = sem_fit(model_ls) fm = fit_measures(solution_ls) - @test all(test_fitmeasures(fm, - solution_lav[:fitmeasures_ls_mean]; - atol = 1e-3, + @test all(test_fitmeasures(fm, + solution_lav[:fitmeasures_ls_mean]; + atol = 1e-3, fitmeasure_names = fitmeasure_names_ls)) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) @@ -351,10 +351,10 @@ end @testset "fitmeasures/se_fiml" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; + @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; atol = 1e-3)) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; + @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 0.002, col = :se, lav_col = :se) end diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 748d232e1..4073c6537 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,7 +1,7 @@ using StructuralEquationModels, Test, FiniteDiff include( - joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), + joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), "test/examples/helper.jl") ) @@ -64,9 +64,9 @@ A =[0 0 0 0 0 0 0 0 0 0 0 1.0 0 0 0 0 0 0 0 0 0 0 0 0 0 :x30 :x31 0] spec = RAMMatrices(; - A = A, - S = S, - F = F, + A = A, + S = S, + F = F, parameters = x, colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) @@ -80,8 +80,8 @@ x = Symbol.("x".*string.(1:38)) M = [:x32; :x33; :x34; :x35; :x36; :x37; :x38; :x35; :x36; :x37; :x38; 0.0; 0.0; 0.0] spec_mean = RAMMatrices(; - A = A, - S = S, + A = A, + S = S, F = F, M = M, parameters = x, diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 7cadbee0c..18c9aef07 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -1,7 +1,7 @@ using StructuralEquationModels, Distributions, Random, Optim, LineSearches include( - joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), + joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), "test/examples/helper.jl") ) @@ -53,11 +53,11 @@ semobserved = SemObservedData(data = x, specification = nothing) loss_ml = SemLoss(SemML(;observed = semobserved, nparams = length(start))) -optimizer = +optimizer = SemOptimizerOptim( - BFGS(;linesearch = BackTracking(order=3), alphaguess = InitialHagerZhang()),# m = 100), + BFGS(;linesearch = BackTracking(order=3), alphaguess = InitialHagerZhang()),# m = 100), Optim.Options( - ;f_tol = 1e-10, + ;f_tol = 1e-10, x_tol = 1.5e-8)) model_ml = Sem(semobserved, imply_ml, loss_ml, optimizer) From 15e0bd29d85980e794dd2db0e5c0ca77ab7b36b9 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 21:51:05 -0700 Subject: [PATCH 067/174] don't import == --- src/frontend/specification/RAMMatrices.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 5325c3677..bda9d87b6 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -8,9 +8,7 @@ struct RAMConstant value end -import Base.== - -function ==(c1::RAMConstant, c2::RAMConstant) +function Base.:(==)(c1::RAMConstant, c2::RAMConstant) res = ( (c1.matrix == c2.matrix) && (c1.index == c2.index) && (c1.value == c2.value) ) return res @@ -340,7 +338,7 @@ function append_partable_rows!(partable::ParameterTable, return nothing end -function ==(mat1::RAMMatrices, mat2::RAMMatrices) +function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) res = ( (mat1.A_ind == mat2.A_ind) && (mat1.S_ind == mat2.S_ind) && (mat1.F_ind == mat2.F_ind) && (mat1.M_ind == mat2.M_ind) && (mat1.parameters == mat2.parameters) && From 78193a77a794bf6d91c7b6fdb6d1d523de273f10 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 10 Mar 2024 21:51:51 -0700 Subject: [PATCH 068/174] don't import push!() --- src/frontend/specification/EnsembleParameterTable.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index df84441ac..0224370e7 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -94,13 +94,11 @@ end # add a row -------------------------------------------------------------------------------- # do we really need this? -import Base.push! - -function push!(partable::EnsembleParameterTable, d::AbstractDict, group) +function Base.push!(partable::EnsembleParameterTable, d::AbstractDict, group) push!(partable.tables[group], d) end -push!(partable::EnsembleParameterTable, d::Nothing, group) = nothing +Base.push!(partable::EnsembleParameterTable, d::Nothing, group) = nothing Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] From 61ac92b41cc87dfda84bcf2cf31e0435acd1e60b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 14:50:28 -0700 Subject: [PATCH 069/174] remove no-op push!() --- src/frontend/specification/EnsembleParameterTable.jl | 2 -- src/frontend/specification/ParameterTable.jl | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 0224370e7..3529381e7 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -98,8 +98,6 @@ function Base.push!(partable::EnsembleParameterTable, d::AbstractDict, group) push!(partable.tables[group], d) end -Base.push!(partable::EnsembleParameterTable, d::Nothing, group) = nothing - Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] ############################################################################################ diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index b2d966f69..0c8110af7 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -185,8 +185,6 @@ function Base.push!(partable::ParameterTable, d::NamedTuple) end end -Base.push!(partable::ParameterTable, d::Nothing) = nothing - ############################################################################################ ### Update Partable from Fitted Model ############################################################################################ From fca77788ec4b5c448e2944a9c59836c4e177e8cc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:12:44 -0700 Subject: [PATCH 070/174] rename Base.sort() to sort_vars() because the ParTable contains rows and columns, it is not clear, what sort() actually sorts. --- src/StructuralEquationModels.jl | 22 +++++++-------- .../specification/EnsembleParameterTable.jl | 19 +++++-------- src/frontend/specification/ParameterTable.jl | 28 +++++++++++++++---- test/examples/multigroup/build_models.jl | 2 +- .../political_democracy.jl | 4 +-- test/unit_tests/sorting.jl | 4 +-- 6 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 0c361c835..bc10b997b 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -78,34 +78,34 @@ include("frontend/fit/standard_errors/bootstrap.jl") -export AbstractSem, - AbstractSemSingle, AbstractSemCollection, Sem, SemFiniteDiff, +export AbstractSem, + AbstractSemSingle, AbstractSemCollection, Sem, SemFiniteDiff, SemEnsemble, MeanStructure, NoMeanStructure, HasMeanStructure, HessianEvaluation, ExactHessian, ApproximateHessian, - SemImply, + SemImply, RAMSymbolic, RAMSymbolicZ, RAM, ImplyEmpty, imply, start_val, start_fabin3, start_simple, start_parameter_table, - SemLoss, + SemLoss, SemLossFunction, SemML, SemFIML, em_mvn, SemLasso, SemRidge, SemConstant, SemWLS, loss, - SemOptimizer, + SemOptimizer, SemOptimizerEmpty, SemOptimizerOptim, SemOptimizerNLopt, NLoptConstraint, optimizer, n_iterations, convergence, - SemObserved, + SemObserved, SemObservedData, SemObservedCovariance, SemObservedMissing, observed, - sem_fit, + sem_fit, SemFit, minimum, solution, sem_summary, - objective!, gradient!, hessian!, objective_gradient!, objective_hessian!, + objective!, gradient!, hessian!, objective_gradient!, objective_hessian!, gradient_hessian!, objective_gradient_hessian!, - ParameterTable, + ParameterTable, EnsembleParameterTable, update_partable!, update_estimate!, update_start!, - Fixed, fixed, Start, start, Label, label, + Fixed, fixed, Start, start, Label, label, sort_vars!, sort_vars, get_identifier_indices, - RAMMatrices, + RAMMatrices, RAMMatrices!, identifier, nparams, fit_measures, diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 3529381e7..8295ff948 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -72,24 +72,19 @@ end ### Additional Methods ############################################################################################ -# Sorting ---------------------------------------------------------------------------------- +# Variables Sorting ------------------------------------------------------------------------ -# Sorting ---------------------------------------------------------------------------------- +function sort_vars!(partables::EnsembleParameterTable) -function sort!(ensemble_partable::EnsembleParameterTable) - - for partable in values(ensemble_partable.tables) - sort!(partable) + for partable in values(partables.tables) + sort_vars!(partable) end - return ensemble_partable + return partables end -function sort(partable::EnsembleParameterTable) - new_partable = deepcopy(partable) - sort!(new_partable) - return new_partable -end +sort_vars(partables::EnsembleParameterTable) = + sort_vars!(deepcopy(partables)) # add a row -------------------------------------------------------------------------------- diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 0c8110af7..cadfbd218 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -132,7 +132,18 @@ end Base.showerror(io::IO, e::CyclicModelError) = print(io, e.msg) -function Base.sort!(partable::ParameterTable) +""" + sort_vars!(partable::ParameterTable) + sort_vars!(partables::EnsembleParameterTable) + +Sort variables in `partable` so that all independent variables are +before the dependent variables and store it in `partable.variables.sorted`. + +If the relations between the variables are acyclic, sorting will +make the resulting `A` matrix in the *RAM* model lower triangular +and allow faster calculations. +""" +function sort_vars!(partable::ParameterTable) vars = [partable.variables.latent; partable.variables.observed] @@ -171,11 +182,16 @@ function Base.sort!(partable::ParameterTable) return partable end -function Base.sort(partable::ParameterTable) - new_partable = deepcopy(partable) - sort!(new_partable) - return new_partable -end +""" + sort_vars(partable::ParameterTable) + sort_vars(partables::EnsembleParameterTable) + +Sort variables in `partable` so that all independent variables are +before the dependent variables, and return the copy of `partable`. + +See [sort_vars!](@ref) for in-place version. +""" +sort_vars(partable::ParameterTable) = sort_vars!(deepcopy(partable)) # add a row -------------------------------------------------------------------------------- diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 36e887486..71aa501cf 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -51,7 +51,7 @@ end # ML estimation - sorted ############################################################################################ -partable_s = sort(partable) +partable_s = sort_vars(partable) specification_s = convert(Dict{Symbol, RAMMatrices}, partable_s) diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 4073c6537..c8c0cdb64 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -159,7 +159,7 @@ spec = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) -sort!(spec) +sort_vars!(spec) partable = spec @@ -192,7 +192,7 @@ spec_mean = ParameterTable(graph, latent_vars = latent_vars, observed_vars = observed_vars) -sort!(spec_mean) +sort_vars!(spec_mean) partable_mean = spec_mean diff --git a/test/unit_tests/sorting.jl b/test/unit_tests/sorting.jl index c31673d08..f3076b8ef 100644 --- a/test/unit_tests/sorting.jl +++ b/test/unit_tests/sorting.jl @@ -1,8 +1,8 @@ ############################################################################ -### test sorting +### test variables sorting ############################################################################ -sort!(partable) +sort_vars!(partable) model_ml_sorted = Sem( specification = partable, From a9d55ee431f5a3914fb866830ee561cb8e8b2a05 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 00:15:27 -0700 Subject: [PATCH 071/174] sort_vars!(ParTable): cleanup --- src/frontend/specification/ParameterTable.jl | 26 ++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index cadfbd218..749a435ac 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -148,12 +148,14 @@ function sort_vars!(partable::ParameterTable) vars = [partable.variables.latent; partable.variables.observed] - is_regression = [(partype == :→) && (from != Symbol("1")) - for (partype, from) in zip(partable.columns.parameter_type, - partable.columns.from)] - - to = partable.columns.to[is_regression] - from = partable.columns.from[is_regression] + # regression edges (excluding intercept) + edges = [(from, to) + for (reltype, from, to) in + zip(partable.columns.parameter_type, + partable.columns.from, + partable.columns.to) + if (reltype == :→) && (from != Symbol("1"))] + sort!(edges, by=last) # sort edges by target sorted_vars = Vector{Symbol}() @@ -162,16 +164,20 @@ function sort_vars!(partable::ParameterTable) acyclic = false for (i, var) in enumerate(vars) - if !(var ∈ to) + # check if var has any incoming edge + eix = searchsortedfirst(edges, (var, var), by=last) + if !(eix <= length(edges) && last(edges[eix]) == var) + # var is source, no edges to it push!(sorted_vars, var) deleteat!(vars, i) - delete_edges = from .!= var - to = to[delete_edges] - from = from[delete_edges] + # remove var outgoing edges + filter!(e -> e[1] != var, edges) acyclic = true + break end end + # if acyclic is false, all vars have incoming edge acyclic || throw(CyclicModelError("your model is cyclic and therefore can not be ordered")) end From 115fc39e3b0fc1de4c991ec94be6b5b790e81c62 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 11 Mar 2024 14:35:43 -0700 Subject: [PATCH 072/174] remove spurious "using SEM" --- src/observed/covariance.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 9f28d112e..598d066d1 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -45,7 +45,7 @@ struct SemObservedCovariance{B, C, D, O} <: SemObserved n_man::D n_obs::O end -using StructuralEquationModels + function SemObservedCovariance(; specification::Union{SemSpecification, Nothing} = nothing, obs_cov, From 25d7b943123c1b4b0b91d618f7f7614cc5bc7b43 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 11 Mar 2024 21:58:28 -0700 Subject: [PATCH 073/174] fix typo --- src/types.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.jl b/src/types.jl index 169936e52..cb60bf5fb 100644 --- a/src/types.jl +++ b/src/types.jl @@ -71,7 +71,7 @@ function SemLoss(functions...; loss_weights = nothing, kwargs...) ) end -# weights for loss functions or models. If the weight is nothing, multiplication returs second argument +# weights for loss functions or models. If the weight is nothing, multiplication returns the second argument struct SemWeight{T} w::T end From ad51f768560dc56fc5acda029180f69a9b360a6c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 2 Apr 2024 18:33:32 -0700 Subject: [PATCH 074/174] add ParamsArray replaces RAMMatrices indices and constants vectors with dedicated class that incapsulate this logic, resulting in overall cleaner interface A_ind, S_ind, M_ind become ParamsArray F_ind becomes SparseMatrixCSC --- src/StructuralEquationModels.jl | 3 +- src/additional_functions/params_array.jl | 169 ++++++++++ .../start_val/start_fabin3.jl | 150 +++++---- .../start_val/start_simple.jl | 22 +- src/frontend/specification/RAMMatrices.jl | 310 +++++++++--------- src/imply/RAM/generic.jl | 61 +--- src/imply/RAM/symbolic.jl | 20 +- 7 files changed, 430 insertions(+), 305 deletions(-) create mode 100644 src/additional_functions/params_array.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index bc10b997b..ee7b4172f 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -16,6 +16,7 @@ include("objective_gradient_hessian.jl") # fitted objects include("frontend/fit/SemFit.jl") # specification of models +include("additional_functions/params_array.jl") include("frontend/specification/ParameterTable.jl") include("frontend/specification/RAMMatrices.jl") include("frontend/specification/EnsembleParameterTable.jl") @@ -51,7 +52,7 @@ include("optimizer/optim.jl") include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") -include("additional_functions/parameters.jl") +#include("additional_functions/parameters.jl") include("additional_functions/start_val/start_val.jl") include("additional_functions/start_val/start_fabin3.jl") include("additional_functions/start_val/start_partable.jl") diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl new file mode 100644 index 000000000..d71e172c9 --- /dev/null +++ b/src/additional_functions/params_array.jl @@ -0,0 +1,169 @@ +""" +Array with partially parameterized elements. +""" +struct ParamsArray{T, N} <: AbstractArray{T, N} + linear_indices::Vector{Int} + param_ptr::Vector{Int} + constants::Vector{Pair{Int, T}} + size::NTuple{N, Int} +end + +ParamsVector{T} = ParamsArray{T, 1} +ParamsMatrix{T} = ParamsArray{T, 2} + +function ParamsArray{T,N}(params_map::AbstractVector{<:AbstractVector{Int}}, + constants::Vector{Pair{Int, T}}, + size::NTuple{N, Int}) where {T,N} + params_ptr = pushfirst!(accumulate((ptr, inds) -> ptr + length(inds), + params_map, init=1), 1) + return ParamsArray{T,N}(reduce(vcat, params_map, init=Vector{Int}()), params_ptr, + constants, size) +end + +function ParamsArray{T,N}(arr::AbstractArray{<:Any, N}, params::AbstractVector{Symbol}; + skip_zeros::Bool = true) where {T,N} + params_index = Dict(param => i for (i, param) in enumerate(params)) + constants = Vector{Pair{Int, T}}() + params_map = [Vector{Int}() for _ in eachindex(params)] + arr_ixs = CartesianIndices(arr) + for (i, val) in pairs(vec(arr)) + ismissing(val) && continue + if isa(val, Number) + (skip_zeros && iszero(val)) || push!(constants, i => val) + else + par_ind = get(params_index, val, nothing) + if !isnothing(par_ind) + push!(params_map[par_ind], i) + else + throw(KeyError("Unrecognized parameter $val at position $(arr_ixs[i])")) + end + end + end + return ParamsArray{T, N}(params_map, constants, size(arr)) +end + +ParamsArray{T}(arr::AbstractArray{<:Any, N}, params::AbstractVector{Symbol}; kwargs...) where {T, N} = + ParamsArray{T,N}(arr, params; kwargs...) + +nparams(arr::ParamsArray) = length(arr.param_ptr) - 1 + +Base.size(arr::ParamsArray) = arr.size +Base.size(arr::ParamsArray, i::Integer) = arr.size[i] + +Base.:(==)(a::ParamsArray, b::ParamsArray) = + return eltype(a) == eltype(b) && + size(a) == size(b) && + a.constants == b.constants && + a.param_ptr == b.param_ptr && + a.linear_indices == b.linear_indices + +# the range of arr.param_ptr indices that correspond to i-th parameter +param_occurences_range(arr::ParamsArray, i::Integer) = + arr.param_ptr[i]:(arr.param_ptr[i + 1] - 1) + +""" + param_occurences(arr::ParamsArray, i::Integer) + +Get the linear indices of the elements in `arr` that equal to the +`i`-th parameter. +""" +param_occurences(arr::ParamsArray, i::Integer) = + view(arr.linear_indices, arr.param_ptr[i]:(arr.param_ptr[i + 1] - 1)) + +""" + materialize!(dest::AbstractArray{<:Any, N}, src::ParamsArray{<:Any, N}, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false) + +Materialize the parameterized array `src` into `dest` by substituting the parameter +references with the parameter values from `param_values`. +""" +function materialize!(dest::AbstractArray{<:Any, N}, src::ParamsArray{<:Any, N}, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false) where N + size(dest) == size(src) || + throw(DimensionMismatch("Parameters ($(size(params_arr))) and destination ($(size(dest))) array sizes don't match")) + nparams(src) == length(param_values) || + throw(DimensionMismatch("Number of values ($(length(param_values))) does not match the number of parameters ($(nparams(src)))")) + Z = eltype(dest) <: Number ? eltype(dest) : eltype(src) + set_zeros && fill!(dest, zero(Z)) + if set_constants + @inbounds for (i, val) in src.constants + dest[i] = val + end + end + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(src, i) + dest[src.linear_indices[j]] = val + end + end + return dest +end + +""" + materialize([T], src::ParamsArray{<:Any, N}, + param_values::AbstractVector{T}) where T + +Materialize the parameterized array `src` into a new array of type `T` +by substituting the parameter references with the parameter values from `param_values`. +""" +materialize(::Type{T}, arr::ParamsArray, param_values::AbstractVector) where T = + materialize!(similar(arr, T), arr, param_values, set_constants=true, set_zeros=true) + +materialize(arr::ParamsArray, param_values::AbstractVector{T}) where T = + materialize(Union{T, eltype(arr)}, arr, param_values) + +function sparse_materialize(::Type{T}, + arr::ParamsMatrix, + param_values::AbstractVector) where T + nparams(arr) == length(param_values) || + throw(DimensionMismatch("Number of values ($(length(param)values))) does not match the number of parameter ($(nparams(arr)))")) + # constant values in sparse matrix + cvals = [T(v) for (_, v) in arr.constants] + # parameter values in sparse matrix + parvals = Vector{T}(undef, length(arr.linear_indices)) + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(arr, i) + parvals[j] = val + end + end + nzixs = [first.(arr.constants); arr.linear_indices] + ixorder = sortperm(nzixs) + nzixs = nzixs[ixorder] + nzvals = [cvals; parvals][ixorder] + arr_ixs = CartesianIndices(size(arr)) + return sparse([arr_ixs[i][1] for i in nzixs], + [arr_ixs[i][2] for i in nzixs], + nzvals, size(arr)...) +end + +sparse_materialize(arr::ParamsArray, params::AbstractVector{T}) where T = + sparse_materialize(Union{T, eltype(arr)}, arr, params) + +# construct length(M)×length(params) sparse matrix of 1s at the positions, +# where the corresponding parameter occurs in the arr +sparse_gradient(::Type{T}, arr::ParamsArray) where T = + SparseMatrixCSC(length(arr), nparams(arr), + arr.param_ptr, arr.linear_indices, ones(T, length(arr.linear_indices))) + +sparse_gradient(arr::ParamsArray{T}) where T = sparse_gradient(T, arr) + +# range of parameters that are referenced in the matrix +function params_range(arr::ParamsArray; allow_gaps::Bool = false) + first_i = findfirst(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1) + last_i = findlast(i -> arr.param_ptr[i+1] > arr.param_ptr[i], 1:nparams(arr)-1) + + if !allow_gaps && !isnothing(first_i) && !isnothing(last_i) + for i in first_i:last_i + if isempty(param_occurences_range(arr, i)) + # TODO show which parameter is missing in which matrix + throw(ErrorException( + "Parameter vector is not partitioned into directed and undirected effects")) + end + end + end + + return first_i:last_i +end diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index 9a4a706b5..d30698bf9 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -51,24 +51,20 @@ end function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) - A_ind, S_ind, F_ind, M_ind, n_par = - ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.F_ind, - ram_matrices.M_ind, + A, S, F, M, n_par = + ram_matrices.A, + ram_matrices.S, + ram_matrices.F, + ram_matrices.M, nparams(ram_matrices) start_val = zeros(n_par) - n_obs = nobserved_vars(ram_matrices) - n_var = nvars(ram_matrices) - n_latent = nlatent_vars(ram_matrices) - - C_indices = CartesianIndices((n_var, n_var)) + F_var2obs = Dict(i => F.rowval[F.colptr[i]] + for i in axes(F, 2) if isobserved_var(ram_matrices, i)) + @assert length(F_var2obs) == size(F, 1) # check in which matrix each parameter appears - indices = Vector{CartesianIndex{2}}(undef, n_par) - #= in_S = length.(S_ind) .!= 0 in_A = length.(A_ind) .!= 0 A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] @@ -86,23 +82,54 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end =# # set undirected parameters in S - for (i, S_ind) in enumerate(S_ind) - for c_ind in C_indices[S_ind] - (c_ind[1] == c_ind[2]) || continue # covariances stay 0 - pos = searchsortedfirst(F_ind, c_ind[1]) - start_val[i] = (pos <= length(F_ind)) && (F_ind[pos] == c_ind[1]) ? - Σ[pos, pos]/2 : 0.05 - break # i-th parameter initialized + S_indices = CartesianIndices(S) + for j in 1:nparams(S) + for lin_ind in param_occurences(S, j) + to, from = Tuple(S_indices[lin_ind]) + if (to == from) # covariances start with 0 + # half of observed variance for observed, 0.05 for latent + obs = get(F_var2obs, to, nothing) + start_val[j] = !isnothing(obs) ? Σ[obs, obs]/2 : 0.05 + break # j-th parameter initialized + end end end # set loadings - constants = ram_matrices.constants - A_ind_c = [linear2cartesian(ind, (n_var, n_var)) for ind in A_ind] + A_indices = CartesianIndices(A) # ind_Λ = findall([is_in_Λ(ind_vec, F_ind) for ind_vec in A_ind_c]) + # collect latent variable indicators in A + # maps latent parameter to the vector of dependent vars + # the 2nd index in the pair specified the parameter index, + # 0 if no parameter (constant), -1 if constant=1 + var2indicators = Dict{Int, Vector{Pair{Int, Int}}}() + for j in 1:nparams(A) + for lin_ind in param_occurences(A, j) + to, from = Tuple(A_indices[lin_ind]) + haskey(F_var2obs, from) && continue # skip observed + obs = get(F_var2obs, to, nothing) + if !isnothing(obs) + indicators = get!(() -> Vector{Pair{Int, Int}}(), var2indicators, from) + push!(indicators, obs => j) + end + end + end + + for (lin_ind, val) in A.constants + iszero(val) && continue # only non-zero loadings + to, from = Tuple(A_indices[lin_ind]) + haskey(F_var2obs, from) && continue # skip observed + obs = get(F_var2obs, to, nothing) + if !isnothing(obs) + indicators = get!(() -> Vector{Pair{Int, Int}}(), var2indicators, from) + push!(indicators, obs => ifelse(isone(val), -1, 0)) # no parameter associated, -1 = reference, 0 = indicator + end + end + + # calculate starting values for parameters of latent regression vars function calculate_lambda(ref::Integer, indicator::Integer, - indicators::AbstractVector{<:Integer}) + indicators::AbstractVector) instruments = filter(i -> (i != ref) && (i != indicator), indicators) if length(instruments) == 1 s13 = Σ[ref, instruments[1]] @@ -117,40 +144,13 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) end end - for i ∈ setdiff(1:n_var, F_ind) - reference = Int64[] - indicators = Int64[] - indicator2parampos = Dict{Int, Int}() - - for (j, Aj_ind_c) in enumerate(A_ind_c) - for ind_c in Aj_ind_c - (ind_c[2] == i) || continue - ind_pos = searchsortedfirst(F_ind, ind_c[1]) - if (ind_pos <= length(F_ind)) && (F_ind[ind_pos] == ind_c[1]) - push!(indicators, ind_pos) - indicator2parampos[ind_pos] = j - end - end - end - - for ram_const in constants - if (ram_const.matrix == :A) && (ram_const.index[2] == i) - ind_pos = searchsortedfirst(F_ind, ram_const.index[1]) - if (ind_pos <= length(F_ind)) && (F_ind[ind_pos] == ram_const.index[1]) - if isone(ram_const.value) - push!(reference, ind_pos) - else - push!(indicators, ind_pos) - # no parameter associated - end - end - end - end - + for (i, indicators) in pairs(var2indicators) + reference = [obs for (obs, param) in indicators if param == -1] + indicator_obs = first.(indicators) # is there at least one reference indicator? if length(reference) > 0 if length(reference) > 1 - if isempty(indicator2parampos) # don't warn if entire column is fixed + if any(((obs, param),) -> param > 0, indicators) # don't warn if entire column is fixed @warn "You have more than 1 scaling indicator for $(ram_matrices.colnames[i])" end ref = reference[1] @@ -158,22 +158,22 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) ref = reference[1] end - for (j, indicator) in enumerate(indicators) - if (indicator != ref) && (parampos = get(indicator2parampos, indicator, 0)) != 0 - start_val[parampos] = calculate_lambda(ref, indicator, indicators) + for (indicator, param) in indicators + if (indicator != ref) && (param > 0) + start_val[param] = calculate_lambda(ref, indicator, indicator_obs) end end # no reference indicator: - elseif length(indicators) > 0 - ref = indicators[1] - λ = Vector{Float64}(undef, length(indicators)); λ[1] = 1.0 - for (j, indicator) in enumerate(indicators) + else + ref = indicator_obs[1] + λ = Vector{Float64}(undef, length(indicator_obs)); λ[1] = 1.0 + for (j, indicator) in enumerate(indicator_obs) if indicator != ref - λ[j] = calculate_lambda(ref, indicator, indicators) + λ[j] = calculate_lambda(ref, indicator, indicator_obs) end end - Σ_λ = Σ[indicators, indicators] + Σ_λ = Σ[indicator_obs, indicator_obs] l₂ = sum(abs2, λ) D = λ*λ' ./ l₂ θ = (I - D.^2)\(diag(Σ_λ - D*Σ_λ*D)) @@ -184,24 +184,22 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) λ .*= sign(Ψ) * sqrt(abs(Ψ)) - for (j, indicator) ∈ enumerate(indicators) - if (parampos = get(indicator2parampos, indicator, 0)) != 0 - start_val[parampos] = λ[j] + for (j, (_, param)) ∈ enumerate(indicators) + if param > 0 + start_val[param] = λ[j] end end - else - @warn "No scaling indicators for $(ram_matrices.colnames[i])" end end - # set means - if !isnothing(M_ind) - for (i, M_ind) in enumerate(M_ind) - if length(M_ind) != 0 - ind = M_ind[1] - pos = searchsortedfirst(F_ind, ind[1]) - if (pos <= length(F_ind)) && (F_ind[pos] == ind[1]) - start_val[i] = μ[pos] + if !isnothing(M) + # set starting values of the observed means + for j in 1:nparams(M) + M_ind = param_occurences(M, j) + if !isempty(M_ind) + obs = get(F_var2obs, M_ind[1], nothing) + if !isnothing(obs) + start_val[j] = μ[obs] end # latent means stay 0 end end @@ -212,4 +210,4 @@ end function is_in_Λ(ind_vec, F_ind) return any(ind -> !(ind[2] ∈ F_ind) && (ind[1] ∈ F_ind), ind_vec) -end \ No newline at end of file +end diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index 6c9fb2018..fee6cc989 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -63,11 +63,11 @@ function start_simple( start_means = 0.0, kwargs...) - A_ind, S_ind, F_ind, M_ind, n_par = - ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.F_ind, - ram_matrices.M_ind, + A, S, F_ind, M, n_par = + ram_matrices.A, + ram_matrices.S, + observed_var_indices(ram_matrices), + ram_matrices.M, nparams(ram_matrices) start_val = zeros(n_par) @@ -77,9 +77,11 @@ function start_simple( C_indices = CartesianIndices((n_var, n_var)) for i in 1:n_par - if length(S_ind[i]) != 0 + Si_ind = param_occurences(S, i) + Ai_ind = param_occurences(A, i) + if length(Si_ind) != 0 # use the first occurence of the parameter to determine starting value - c_ind = C_indices[S_ind[i][1]] + c_ind = C_indices[Si_ind[1]] if c_ind[1] == c_ind[2] if c_ind[1] ∈ F_ind start_val[i] = start_variances_observed @@ -97,14 +99,14 @@ function start_simple( start_val[i] = start_covariances_obs_lat end end - elseif length(A_ind[i]) != 0 - c_ind = C_indices[A_ind[i][1]] + elseif length(Ai_ind) != 0 + c_ind = C_indices[Ai_ind[1]] if (c_ind[1] ∈ F_ind) & !(c_ind[2] ∈ F_ind) start_val[i] = start_loadings else start_val[i] = start_regressions end - elseif !isnothing(M_ind) && (length(M_ind[i]) != 0) + elseif !isnothing(M) && (length(param_occurences(M, i)) != 0) start_val[i] = start_means end end diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index bda9d87b6..f899c8066 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -1,75 +1,39 @@ -############################################################################################ -### Constants -############################################################################################ - -struct RAMConstant - matrix::Symbol - index::CartesianIndex - value -end - -function Base.:(==)(c1::RAMConstant, c2::RAMConstant) - res = ( (c1.matrix == c2.matrix) && (c1.index == c2.index) && - (c1.value == c2.value) ) - return res -end - -function append_RAMConstants!(constants::AbstractVector{RAMConstant}, - mtx_name::Symbol, mtx::AbstractArray; - skip_zeros::Bool = true) - for (index, val) in pairs(mtx) - if isa(val, Number) && !(skip_zeros && iszero(val)) - push!(constants, RAMConstant(mtx_name, index, val)) - end - end - return constants -end - -function set_RAMConstant!(A, S, M, rc::RAMConstant) - if rc.matrix == :A - A[rc.index] = rc.value - elseif rc.matrix == :S - S[rc.index] = rc.value - S[rc.index[2], rc.index[1]] = rc.value # symmetric - elseif rc.matrix == :M - M[rc.index] = rc.value - end -end - -function set_RAMConstants!(A, S, M, rc_vec::Vector{RAMConstant}) - for rc in rc_vec set_RAMConstant!(A, S, M, rc) end -end ############################################################################################ ### Type ############################################################################################ -# map from parameter index to linear indices of matrix/vector positions where it occurs -AbstractArrayParamsMap = AbstractVector{<:AbstractVector{<:Integer}} -ArrayParamsMap = Vector{Vector{Int}} - struct RAMMatrices <: SemSpecification - A_ind::ArrayParamsMap - S_ind::ArrayParamsMap - F_ind::Vector{Int} - M_ind::Union{ArrayParamsMap, Nothing} + A::ParamsMatrix{Float64} + S::ParamsMatrix{Float64} + F::SparseMatrixCSC{Float64} + M::Union{ParamsVector{Float64}, Nothing} parameters::Vector{Symbol} - colnames::Vector{Symbol} - constants::Vector{RAMConstant} - size_F::Tuple{Int, Int} + colnames::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_colnames()) end -nparams(ram::RAMMatrices) = length(ram.A_ind) -nvars(ram::RAMMatrices) = ram.size_F[2] -nobserved_vars(ram::RAMMatrices) = ram.size_F[1] +nparams(ram::RAMMatrices) = nparams(ram.A) +nvars(ram::RAMMatrices) = size(ram.F, 2) +nobserved_vars(ram::RAMMatrices) = size(ram.F, 1) nlatent_vars(ram::RAMMatrices) = nvars(ram) - nobserved_vars(ram) +isobserved_var(ram::RAMMatrices, i::Integer) = + ram.F.colptr[i+1] > ram.F.colptr[i] +islatent_var(ram::RAMMatrices, i::Integer) = + ram.F.colptr[i+1] == ram.F.colptr[i] + +observed_var_indices(ram::RAMMatrices) = + [i for i in axes(ram.F, 2) if isobserved_var(ram, i)] +latent_var_indices(ram::RAMMatrices) = + [i for i in axes(ram.F, 2) if islatent_var(ram, i)] + function observed_vars(ram::RAMMatrices) if isnothing(ram.colnames) @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" return nothing else - return view(ram.colnames, ram.F_ind) + return [col for (i, col) in enumerate(ram.colnames) + if isobserved_var(ram, i)] end end @@ -78,7 +42,8 @@ function latent_vars(ram::RAMMatrices) @warn "Your RAMMatrices do not contain column names. Please make sure the order of variables in your data is correct!" return nothing else - return view(ram.colnames, setdiff(eachindex(ram.colnames), ram.F_ind)) + return [col for (i, col) in enumerate(ram.colnames) + if islatent_var(ram, i)] end end @@ -89,23 +54,27 @@ end function RAMMatrices(; A::AbstractMatrix, S::AbstractMatrix, F::AbstractMatrix, M::Union{AbstractVector, Nothing} = nothing, parameters::AbstractVector{Symbol}, - colnames::AbstractVector{Symbol}) - ncols = length(colnames) + colnames::Union{AbstractVector{Symbol}, Nothing} = nothing) + ncols = size(A, 2) + if !isnothing(colnames) + length(colnames) == ncols || throw(DimensionMismatch("colnames length ($(length(colnames))) does not match the number of columns in A ($ncols)")) + end size(A, 1) == size(A, 2) || throw(DimensionMismatch("A must be a square matrix")) size(S, 1) == size(S, 2) || throw(DimensionMismatch("S must be a square matrix")) - size(A, 2) == ncols || throw(DimensionMismatch("A should have as many rows and columns as colnames length ($(length(colnames))), $(size(A)) found")) - size(S, 2) == ncols || throw(DimensionMismatch("S should have as many rows and columns as colnames length ($(length(colnames))), $(size(S)) found")) - size(F, 2) == ncols || throw(DimensionMismatch("F should have as many columns as colnames length ($(length(colnames))), $(size(F, 2)) found")) - A_indices = array_parameters_map_linear(parameters, A) - S_indices = array_parameters_map_linear(parameters, S) - M_indices = !isnothing(M) ? array_parameters_map_linear(parameters, M) : nothing - F_indices = [i for (i, col) in zip(axes(F, 2), eachcol(F)) if any(isone, col)] - constants = Vector{RAMConstant}() - append_RAMConstants!(constants, :A, A) - append_RAMConstants!(constants, :S, S) - isnothing(M) || append_RAMConstants!(constants, :M, M) - return RAMMatrices(A_indices, S_indices, F_indices, M_indices, - parameters, colnames, constants, size(F)) + size(A, 2) == ncols || throw(DimensionMismatch("A should have as many rows and columns as colnames length ($ncols), $(size(A)) found")) + size(S, 2) == ncols || throw(DimensionMismatch("S should have as many rows and columns as colnames length ($ncols), $(size(S)) found")) + size(F, 2) == ncols || throw(DimensionMismatch("F should have as many columns as colnames length ($ncols), $(size(F, 2)) found")) + if !isnothing(M) + length(M) == ncols || throw(DimensionMismatch("M should have as many elements as colnames length ($ncols), $(length(M)) found")) + end + A = ParamsMatrix{Float64}(A, parameters) + S = ParamsMatrix{Float64}(S, parameters) + M = !isnothing(M) ? ParamsVector{Float64}(M, parameters) : nothing + spF = sparse(F) + if any(!isone, spF.nzval) + throw(ArgumentError("F should contain only 0s and 1s")) + end + return RAMMatrices(A, S, F, M, parameters, colnames) end ############################################################################################ @@ -130,33 +99,38 @@ function RAMMatrices(partable::ParameterTable; n_observed = length(partable.variables.observed) n_latent = length(partable.variables.latent) - n_node = n_observed + n_latent - - # F indices - F_ind = length(partable.variables.sorted) != 0 ? - findall(∈(Set(partable.variables.observed)), - partable.variables.sorted) : - 1:n_observed + n_vars = n_observed + n_latent + + # colnames (variables) + # and F indices (map from each observed column to its variable index) + if length(partable.variables.sorted) != 0 + @assert length(partable.variables.sorted) == nvars(partable) + colnames = copy(partable.variables.sorted) + F_inds = findall(∈(Set(partable.variables.observed)), + colnames) + else + colnames = [partable.variables.observed; + partable.variables.latent] + F_inds = 1:n_observed + end # indices of the colnames - colnames = length(partable.variables.sorted) != 0 ? - copy(partable.variables.sorted) : - [partable.variables.observed; - partable.variables.latent] cols_index = Dict(col => i for (i, col) in enumerate(colnames)) # fill Matrices # known_labels = Dict{Symbol, Int64}() - A_ind = [Vector{Int64}() for _ in 1:length(params)] - S_ind = [Vector{Int64}() for _ in 1:length(params)] - + T = nonmissingtype(eltype(partable.columns.value_fixed)) + A_inds = [Vector{Int64}() for _ in 1:length(params)] + A_lin_ixs = LinearIndices((n_vars, n_vars)) + S_inds = [Vector{Int64}() for _ in 1:length(params)] + S_lin_ixs = LinearIndices((n_vars, n_vars)) + A_consts = Vector{Pair{Int, T}}() + S_consts = Vector{Pair{Int, T}}() # is there a meanstructure? - M_ind = any(==(Symbol("1")), partable.columns.from) ? + M_inds = any(==(Symbol("1")), partable.columns.from) ? [Vector{Int64}() for _ in 1:length(params)] : nothing - - # handle constants - constants = Vector{RAMConstant}() + M_consts = !isnothing(M_inds) ? Vector{Pair{Int, T}}() : nothing for row in partable @@ -165,35 +139,56 @@ function RAMMatrices(partable::ParameterTable; if !row.free if (row.parameter_type == :→) && (row.from == Symbol("1")) - push!(constants, RAMConstant(:M, row_ind, row.value_fixed)) + push!(M_consts, row_ind => row.value_fixed) elseif (row.parameter_type == :→) - push!(constants, RAMConstant(:A, CartesianIndex(row_ind, col_ind), row.value_fixed)) + push!(A_consts, A_lin_ixs[CartesianIndex(row_ind, col_ind)] => row.value_fixed) elseif (row.parameter_type == :↔) - push!(constants, RAMConstant(:S, CartesianIndex(row_ind, col_ind), row.value_fixed)) + push!(S_consts, S_lin_ixs[CartesianIndex(row_ind, col_ind)] => row.value_fixed) + if row_ind != col_ind # symmetric + push!(S_consts, S_lin_ixs[CartesianIndex(col_ind, row_ind)] => row.value_fixed) + end else error("Unsupported parameter type: $(row.parameter_type)") end else par_ind = params_index[row.identifier] if (row.parameter_type == :→) && (row.from == Symbol("1")) - push!(M_ind[par_ind], row_ind) + push!(M_inds[par_ind], row_ind) elseif row.parameter_type == :→ - push!(A_ind[par_ind], row_ind + (col_ind-1)*n_node) + push!(A_inds[par_ind], A_lin_ixs[CartesianIndex(row_ind, col_ind)]) elseif row.parameter_type == :↔ - push!(S_ind[par_ind], row_ind + (col_ind-1)*n_node) - if row_ind != col_ind - push!(S_ind[par_ind], col_ind + (row_ind-1)*n_node) + push!(S_inds[par_ind], S_lin_ixs[CartesianIndex(row_ind, col_ind)]) + if row_ind != col_ind # symmetric + push!(S_inds[par_ind], S_lin_ixs[CartesianIndex(col_ind, row_ind)]) end else error("Unsupported parameter type: $(row.parameter_type)") end end - + end + # sort linear indices + for A_ind in A_inds + sort!(A_ind) + end + for S_ind in S_inds + unique!(sort!(S_ind)) # also symmetric duplicates + end + if !isnothing(M_inds) + for M_ind in M_inds + sort!(M_ind) + end + end + sort!(A_consts, by=first) + sort!(S_consts, by=first) + if !isnothing(M_consts) + sort!(M_consts, by=first) end - return RAMMatrices(A_ind, S_ind, F_ind, M_ind, - params, colnames, constants, - (n_observed, n_node)) + return RAMMatrices(ParamsMatrix{T}(A_inds, A_consts, (n_vars, n_vars)), + ParamsMatrix{T}(S_inds, S_consts, (n_vars, n_vars)), + sparse(1:n_observed, F_inds, ones(T, length(F_inds)), n_observed, n_vars), + !isnothing(M_inds) ? ParamsVector{T}(M_inds, M_consts, (n_vars,)) : nothing, + params, colnames) end Base.convert(::Type{RAMMatrices}, partable::ParameterTable) = RAMMatrices(partable) @@ -206,26 +201,19 @@ function ParameterTable(ram_matrices::RAMMatrices) colnames = ram_matrices.colnames - partable = ParameterTable(observed_vars = colnames[ram_matrices.F_ind], + partable = ParameterTable(observed_vars = colnames[ram_matrices.F.rowval], latent_vars = colnames[setdiff(eachindex(colnames), - ram_matrices.F_ind)]) + ram_matrices.F.rowval)]) - position_names = Dict{Int64, Symbol}(1:length(colnames) .=> colnames) + position_names = Dict{Int, Symbol}(1:length(colnames) .=> colnames) - # constants - for c in ram_matrices.constants - push!(partable, partable_row(c, position_names)) - end - - # parameters - for (i, par) in enumerate(ram_matrices.parameters) - append_partable_rows!( - partable, position_names, - par, i, - ram_matrices.A_ind, - ram_matrices.S_ind, - ram_matrices.M_ind, - ram_matrices.size_F[2]) + append_rows!(partable, ram_matrices.A, :A, + ram_matrices.parameters, position_names) + append_rows!(partable, ram_matrices.S, :S, + ram_matrices.parameters, position_names, skip_symmetric=true) + if !isnothing(ram_matrices.M) + append_rows!(partable, ram_matrices.M, :M, + ram_matrices.parameters, position_names) end return partable @@ -275,63 +263,64 @@ function matrix_to_parameter_type(matrix::Symbol) end end -partable_row(c::RAMConstant, position_names::AbstractDict) = ( - from = position_names[c.index[2]], - parameter_type = matrix_to_parameter_type(c.matrix), - to = position_names[c.index[1]], - free = false, - value_fixed = c.value, - start = 0.0, - estimate = 0.0, - identifier = :const - ) - -function partable_row(par::Symbol, position_names::AbstractDict, - index::Integer, matrix::Symbol, n_nod::Integer) +function partable_row(val, index, matrix::Symbol, + position_names::AbstractDict; + free::Bool = true) # variable names if matrix == :M from = Symbol("1") to = position_names[index] else - cart_index = linear2cartesian(index, (n_nod, n_nod)) - - from = position_names[cart_index[2]] - to = position_names[cart_index[1]] + from = position_names[index[2]] + to = position_names[index[1]] end return ( from = from, parameter_type = matrix_to_parameter_type(matrix), to = to, - free = true, - value_fixed = 0.0, + free = free, + value_fixed = free ? 0.0 : val, start = 0.0, estimate = 0.0, - identifier = par) + identifier = free ? val : :const) end -function append_partable_rows!(partable::ParameterTable, - position_names, par::Symbol, par_index::Integer, - A_ind, S_ind, M_ind, n_nod::Integer) - for ind in A_ind[par_index] - push!(partable, partable_row(par, position_names, ind, :A, n_nod)) - end - - visited_S_indices = Set{Int}() - for ind in S_ind[par_index] - if ind ∉ visited_S_indices - push!(partable, partable_row(par, position_names, ind, :S, n_nod)) - # mark index and its symmetric as visited - push!(visited_S_indices, ind) - cart_index = linear2cartesian(ind, (n_nod, n_nod)) - push!(visited_S_indices, cartesian2linear(CartesianIndex(cart_index[2], cart_index[1]), (n_nod, n_nod))) +function append_rows!(partable::ParameterTable, + arr::ParamsArray, arr_name::Symbol, + parameters::AbstractVector, + position_names; + skip_symmetric::Bool = false) + nparams(arr) == length(params) || + throw(ArgumentError("Length of parameters vector does not match the number of parameters in the matrix")) + arr_ixs = eachindex(arr) + + # add parameters + visited_indices = Set{eltype(arr_ixs)}() + for (i, par) in enumerate(parameters) + for j in param_occurences_range(arr, i) + arr_ix = arr_ixs[arr.linear_indices[j]] + skip_symmetric && (arr_ix ∈ visited_indices) && continue + + push!(partable, partable_row(par, arr_ix, arr_name, position_names, free=true)) + if skip_symmetric + # mark index and its symmetric as visited + push!(visited_indices, arr_ix) + push!(visited_indices, CartesianIndex(arr_ix[2], arr_ix[1])) + end end end - if !isnothing(M_ind) - for ind in M_ind[par_index] - push!(partable, partable_row(par, position_names, ind, :M, n_nod)) + # add constants + for (i, val) in arr.constants + arr_ix = arr_ixs[i] + skip_symmetric && (arr_ix ∈ visited_indices) && continue + push!(partable, partable_row(val, arr_ix, arr_name, position_names, free=false)) + if skip_symmetric + # mark index and its symmetric as visited + push!(visited_indices, arr_ix) + push!(visited_indices, CartesianIndex(arr_ix[2], arr_ix[1])) end end @@ -339,10 +328,9 @@ function append_partable_rows!(partable::ParameterTable, end function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) - res = ( (mat1.A_ind == mat2.A_ind) && (mat1.S_ind == mat2.S_ind) && - (mat1.F_ind == mat2.F_ind) && (mat1.M_ind == mat2.M_ind) && + res = ( (mat1.A == mat2.A) && (mat1.S == mat2.S) && + (mat1.F == mat2.F) && (mat1.M == mat2.M) && (mat1.parameters == mat2.parameters) && - (mat1.colnames == mat2.colnames) && (mat1.size_F == mat2.size_F) && - (mat1.constants == mat2.constants) ) + (mat1.colnames == mat2.colnames) ) return res end diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 5432d86d5..61da17d8b 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -65,7 +65,7 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S1, S2, S3, D} <: SemImply{MS, ExactHessian} +mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, M1, M2, M3, M4, S1, S2, S3, D} <: SemImply{MS, ExactHessian} Σ::A1 A::A2 S::A3 @@ -75,10 +75,6 @@ mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, I1, I2, I3, M1, M2, M3, M4, S ram_matrices::V2 - A_indices::I1 - S_indices::I2 - M_indices::I3 - F⨉I_A⁻¹::M1 F⨉I_A⁻¹S::M2 I_A::M3 @@ -110,21 +106,14 @@ function RAM(; n_par = nparams(ram_matrices) n_obs = nobserved_vars(ram_matrices) n_var = nvars(ram_matrices) - F = zeros(ram_matrices.size_F); F[CartesianIndex.(1:n_var, ram_matrices.F_ind)] .= 1.0 - - # get indices - A_indices = copy(ram_matrices.A_ind) - S_indices = copy(ram_matrices.S_ind) - M_indices = !isnothing(ram_matrices.M_ind) ? copy(ram_matrices.M_ind) : nothing #preallocate arrays - A_pre = zeros(n_var, n_var) - S_pre = zeros(n_var, n_var) - M_pre = !isnothing(M_indices) ? zeros(n_var) : nothing + nan_params = fill(NaN, n_par) + A_pre = materialize(ram_matrices.A, nan_params) + S_pre = materialize(ram_matrices.S, nan_params) + F = Matrix(ram_matrices.F) - set_RAMConstants!(A_pre, S_pre, M_pre, ram_matrices.constants) - - A_pre = check_acyclic(A_pre, n_par, A_indices) + A_pre = check_acyclic(A_pre, ram_matrices.A) # pre-allocate some matrices Σ = zeros(n_obs, n_obs) @@ -133,8 +122,8 @@ function RAM(; I_A = similar(A_pre) if gradient_required - ∇A = matrix_gradient(A_indices, n_var^2) - ∇S = matrix_gradient(S_indices, n_var^2) + ∇A = sparse_gradient(ram_matrices.A) + ∇S = sparse_gradient(ram_matrices.S) else ∇A = nothing ∇S = nothing @@ -143,11 +132,11 @@ function RAM(; # μ if meanstructure MS = HasMeanStructure - ∇M = gradient ? matrix_gradient(M_indices, n_var) : nothing + M_pre = materialize(ram_matrices.M, nan_params) + ∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing μ = zeros(n_obs) else MS = NoMeanStructure - M_indices = nothing M_pre = nothing μ = nothing ∇M = nothing @@ -163,10 +152,6 @@ function RAM(; ram_matrices, - A_indices, - S_indices, - M_indices, - F⨉I_A⁻¹, F⨉I_A⁻¹S, I_A, @@ -185,15 +170,11 @@ end ############################################################################################ function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, parameters) - - fill_A_S_M!( - imply.A, - imply.S, - imply.M, - imply.A_indices, - imply.S_indices, - imply.M_indices, - parameters) + materialize!(imply.A, imply.ram_matrices.A, parameters) + materialize!(imply.S, imply.ram_matrices.S, parameters) + if !isnothing(imply.M) + materialize!(imply.M, imply.ram_matrices.M, parameters) + end @inbounds for (j, I_Aj, Aj) in zip(axes(imply.A, 2), eachcol(imply.I_A), eachcol(imply.A)) for i in axes(imply.A, 1) @@ -237,15 +218,9 @@ end ### additional functions ############################################################################################ -function check_acyclic(A_pre, n_par, A_indices) - # fill copy of A-matrix with random parameters - A_rand = copy(A_pre) - randpar = rand(n_par) - - fill_matrix!( - A_rand, - A_indices, - randpar) +function check_acyclic(A_pre::AbstractMatrix, A::ParamsMatrix) + # fill copy of A with random parameters + A_rand = materialize(A, rand(nparams(A))) # check if the model is acyclic acyclic = isone(det(I-A_rand)) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 3eed25a8a..b55a907c8 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -99,23 +99,15 @@ function RAMSymbolic(; ram_matrices = convert(RAMMatrices, specification) n_par = nparams(ram_matrices) - n_obs = nobserved_vars(ram_matrices) - n_var = nvars(ram_matrices) - par = (Symbolics.@variables θ[1:n_par])[1] - A = zeros(Num, n_var, n_var) - S = zeros(Num, n_var, n_var) - !isnothing(ram_matrices.M_ind) ? M = zeros(Num, n_var) : M = nothing - F = zeros(ram_matrices.size_F); F[CartesianIndex.(1:n_obs, ram_matrices.F_ind)] .= 1.0 - - set_RAMConstants!(A, S, M, ram_matrices.constants) - fill_A_S_M!(A, S, M, ram_matrices.A_ind, ram_matrices.S_ind, ram_matrices.M_ind, par) - - A, S, F = sparse(A), sparse(S), sparse(F) + A = sparse_materialize(Num, ram_matrices.A, par) + S = sparse_materialize(Num, ram_matrices.S, par) + M = !isnothing(ram_matrices.M) ? materialize(Num, ram_matrices.M, par) : nothing + F = ram_matrices.F - if !isnothing(loss_types) - any(loss_types .<: SemWLS) ? vech = true : nothing + if !isnothing(loss_types) && any(T -> T<:SemWLS, loss_types) + vech = true end # Σ From d6e90e5cfc46015d96ce22a5028f77a14901ec75 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 16:42:59 -0700 Subject: [PATCH 075/174] vars(RAMMatrices) --- src/frontend/specification/RAMMatrices.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index f899c8066..0a611b54c 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -17,6 +17,8 @@ nvars(ram::RAMMatrices) = size(ram.F, 2) nobserved_vars(ram::RAMMatrices) = size(ram.F, 1) nlatent_vars(ram::RAMMatrices) = nvars(ram) - nobserved_vars(ram) +vars(ram::RAMMatrices) = ram.colnames + isobserved_var(ram::RAMMatrices, i::Integer) = ram.F.colptr[i+1] > ram.F.colptr[i] islatent_var(ram::RAMMatrices, i::Integer) = From 39e9d0eba156efd30716633a4ca1d90f693843b6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 11 Mar 2024 22:07:56 -0700 Subject: [PATCH 076/174] reorder_obs_cov/mean(): cleanup --- src/observed/covariance.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 598d066d1..b2d58e949 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -118,9 +118,8 @@ function reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) if spec_colnames == obs_colnames return obs_cov else - new_position = [findall(x .== obs_colnames)[1] for x in spec_colnames] - indices = reshape([CartesianIndex(i, j) for j in new_position for i in new_position], size(obs_cov, 1), size(obs_cov, 1)) - obs_cov = obs_cov[indices] + new_position = [findfirst(==(x), obs_colnames) for x in spec_colnames] + obs_cov = obs_cov[new_position, new_position] return obs_cov end end @@ -132,7 +131,7 @@ function reorder_obs_mean(obs_mean, spec_colnames, obs_colnames) if spec_colnames == obs_colnames return obs_mean else - new_position = [findall(x .== obs_colnames)[1] for x in spec_colnames] + new_position = [findfirst(==(x), obs_colnames) for x in spec_colnames] obs_mean = obs_mean[new_position] return obs_mean end From 30343ce6e71b95554a8f67b66faa947b1e1e7f98 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 16:01:51 -0700 Subject: [PATCH 077/174] param_values(ParTable) --- src/frontend/specification/ParameterTable.jl | 52 +++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 749a435ac..60baa397c 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -299,4 +299,54 @@ function update_se_hessian!( hessian = :finitediff) se = se_hessian(sem_fit; hessian = hessian) return update_partable!(partable, sem_fit, se, :se) -end \ No newline at end of file +end + +""" + param_values!(out::AbstractVector, partable::ParameterTable, + col::Symbol = :estimate) + +Extract parameter values from the `col` column of `partable` +into the `out` vector. + +The `out` vector should be of `nparams(partable)` length. +The *i*-th element of the `out` vector will contain the +value of the *i*-th parameter from `params(partable)`. + +Note that the function combines the duplicate occurences of the +same parameter in `partable` and will raise an error if the +values do not match. +""" +function param_values!(out::AbstractVector, partable::ParameterTable, + col::Symbol = :estimate) + (length(out) == nparams(partable)) || + throw(DimensionMismatch("The length of parameter values vector ($(length(out))) does not match the number of parameters ($(nparams(partable)))")) + param_index = Dict(param => i for (i, param) in enumerate(params(partable))) + param_values_col = partable.columns[col] + for (i, param) in enumerate(partable.columns.param) + (param == :const) && continue + param_ind = get(param_index, param, nothing) + @assert !isnothing(param_ind) "Parameter table contains unregistered parameter :$param at row #$i" + val = param_values_col[i] + if !isnan(out[param_ind]) + @assert out[param_ind] ≈ val atol=1E-10 "Parameter :$param value at row #$i ($val) differs from the earlier encountered value ($(out[param_ind]))" + else + out[param_ind] = val + end + end + return out +end + +""" + param_values(out::AbstractVector, col::Symbol = :estimate) + +Extract parameter values from the `col` column of `partable`. + +Returns the values vector. The *i*-th element corresponds to +the value of *i*-th parameter from `params(partable)`. + +Note that the function combines the duplicate occurences of the +same parameter in `partable` and will raise an error if the +values do not match. +""" +param_values(partable::ParameterTable, col::Symbol = :estimate) = + param_values!(fill(NaN, nparams(partable)), partable, col) From b9591cc91749d0f49c36cb011d5134c2ac4a2ea6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 16:03:06 -0700 Subject: [PATCH 078/174] lavaan_param_values(lav_fit, partable) --- src/frontend/specification/ParameterTable.jl | 101 +++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 60baa397c..07ea2c630 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -350,3 +350,104 @@ values do not match. """ param_values(partable::ParameterTable, col::Symbol = :estimate) = param_values!(fill(NaN, nparams(partable)), partable, col) + +""" + lavaan_param_values!(out::AbstractVector, partable_lav, + partable::ParameterTable, + lav_col::Symbol = :est, lav_group = nothing) + +Extract parameter values from the `partable_lav` lavaan model that +match the parameters of `partable` into the `out` vector. + +The method sets the *i*-th element of the `out` vector to +the value of *i*-th parameter from `params(partable)`. + +Note that the lavaan and `partable` models are matched by the +the names of variables in the tables (`from` and `to` columns) +as well as the type of their relationship (`relation` column), +and not by the names of the model parameters. +""" +function lavaan_param_values!(out::AbstractVector, + partable_lav, partable::ParameterTable, + lav_col::Symbol = :est, lav_group = nothing) + + # find indices of all df row where f is true + findallrows(f::Function, df) = findall(f(r) for r in eachrow(df)) + + (length(out) == nparams(partable)) || throw(DimensionMismatch("The length of parameter values vector ($(length(out))) does not match the number of parameters ($(nparams(partable)))")) + partable_mask = findall(partable.columns[:free]) + param_index = Dict(param => i for (i, param) in enumerate(params(partable))) + + lav_values = partable_lav[:, lav_col] + for (from, to, type, id) in + zip([view(partable.columns[k], partable_mask) + for k in [:from, :to, :parameter_type, :param]]...) + + lav_ind = nothing + + if from == Symbol("1") + lav_ind = findallrows(r -> r[:lhs] == String(to) && r[:op] == "~1" && + (isnothing(lav_group) || r[:group] == lav_group), partable_lav) + else + if type == :↔ + lav_type = "~~" + elseif type == :→ + if (from ∈ partable.variables.latent) && (to ∈ partable.variables.observed) + lav_type = "=~" + else + lav_type = "~" + from, to = to, from + end + end + + if lav_type == "~~" + lav_ind = findallrows(r -> ((r[:lhs] == String(from) && r[:rhs] == String(to)) || + (r[:lhs] == String(to) && r[:rhs] == String(from))) && + r[:op] == lav_type && + (isnothing(lav_group) || r[:group] == lav_group), + partable_lav) + else + lav_ind = findallrows(r -> r[:lhs] == String(from) && r[:rhs] == String(to) && r[:op] == lav_type && + (isnothing(lav_group) || r[:group] == lav_group), + partable_lav) + end + end + + if length(lav_ind) == 0 + throw(ErrorException("Parameter $id ($from $type $to) could not be found in the lavaan solution")) + elseif length(lav_ind) > 1 + throw(ErrorException("At least one parameter was found twice in the lavaan solution")) + end + + param_ind = param_index[id] + param_val = lav_values[lav_ind[1]] + if isnan(out[param_ind]) + out[param_ind] = param_val + else + @assert out[param_ind] ≈ param_val atol=1E-10 "Parameter :$id value at row #$lav_ind ($param_val) differs from the earlier encountered value ($(out[param_ind]))" + end + end + + return out +end + +""" + lavaan_param_values(partable_lav, partable::ParameterTable, + lav_col::Symbol = :est, lav_group = nothing) + +Extract parameter values from the `partable_lav` lavaan model that +match the parameters of `partable`. + +The `out` vector should be of `nparams(partable)` length. +The *i*-th element of the `out` vector will contain the +value of the *i*-th parameter from `params(partable)`. + +Note that the lavaan and `partable` models are matched by the +the names of variables in the tables (`from` and `to` columns), +and the type of their relationship (`relation` column), +but not by the ids of the model parameters. +""" +lavaan_param_values(partable_lav, partable::ParameterTable, + lav_col::Symbol = :est, lav_group = nothing) = + lavaan_param_values!(fill(NaN, nparams(partable)), + partable_lav, partable, lav_col, lav_group) From 958d65a41188594ebb2f535c0d67ffa943c9c4a7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 21:57:11 -0700 Subject: [PATCH 079/174] test_gradient(): do tests inside --- test/examples/helper.jl | 13 ++++++------- test/examples/multigroup/build_models.jl | 10 +++++----- test/examples/political_democracy/by_parts.jl | 8 ++++---- test/examples/political_democracy/constructor.jl | 8 ++++---- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 6d4dc9d18..155c357cb 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,17 +1,16 @@ function test_gradient(model, parameters; rtol = 1e-10, atol = 0) - true_grad = FiniteDiff.finite_difference_gradient(x -> objective!(model, x)[1], parameters) - gradient = similar(parameters); gradient .= 1.0 + true_grad = FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), parameters) + gradient = similar(parameters) # F and G + fill!(gradient, NaN) gradient!(gradient, model, parameters) - correct1 = isapprox(gradient, true_grad; rtol = rtol, atol = atol) + @test gradient ≈ true_grad rtol = rtol atol = atol # only G - gradient .= 1.0 + fill!(gradient, NaN) objective_gradient!(gradient, model, parameters) - correct2 = isapprox(gradient, true_grad; rtol = rtol, atol = atol) - - return correct1 & correct2 + @test gradient ≈ true_grad rtol = rtol atol = atol end function test_hessian(model, parameters; rtol = 1e-4, atol = 0) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 71aa501cf..feab0f6b2 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -19,7 +19,7 @@ model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) # gradients @testset "ml_gradients_multigroup" begin - @test test_gradient(model_ml_multigroup, start_test; atol = 1e-9) + test_gradient(model_ml_multigroup, start_test; atol = 1e-9) end # fit @@ -74,7 +74,7 @@ model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) # gradients @testset "ml_gradients_multigroup | sorted" begin - @test test_gradient(model_ml_multigroup, start_test; atol = 1e-2) + test_gradient(model_ml_multigroup, start_test; atol = 1e-2) end grad = similar(start_test) @@ -147,7 +147,7 @@ model_g2 = SemFiniteDiff( model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) @testset "gradients_user_defined_loss" begin - @test test_gradient(model_ml_multigroup, start_test; atol = 1e-9) + test_gradient(model_ml_multigroup, start_test; atol = 1e-9) end # fit @@ -181,7 +181,7 @@ model_ls_g2 = Sem( model_ls_multigroup = SemEnsemble(model_ls_g1, model_ls_g2; optimizer = semoptimizer) @testset "ls_gradients_multigroup" begin - @test test_gradient(model_ls_multigroup, start_test; atol = 1e-9) + test_gradient(model_ls_multigroup, start_test; atol = 1e-9) end @testset "ls_solution_multigroup" begin @@ -251,7 +251,7 @@ start_test = [ fill(0.01, 9)] @testset "fiml_gradients_multigroup" begin - @test test_gradient(model_ml_multigroup, start_test; atol = 1e-7) + test_gradient(model_ml_multigroup, start_test; atol = 1e-7) end diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 22caf33c9..0894225e0 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -51,7 +51,7 @@ model_names = ["ml", "ls_sym", "ridge", "constant", "ml_sym", "ml_weighted"] for (model, name) in zip(models, model_names) try @testset "$(name)_gradient" begin - @test test_gradient(model, start_test; rtol = 1e-9) + test_gradient(model, start_test; rtol = 1e-9) end catch end @@ -218,7 +218,7 @@ model_names = ["ml", "ls_sym", "ml_sym"] for (model, name) in zip(models, model_names) try @testset "$(name)_gradient_mean" begin - @test test_gradient(model, start_test_mean; rtol = 1e-9) + test_gradient(model, start_test_mean; rtol = 1e-9) end catch end @@ -287,11 +287,11 @@ model_ml_sym = Sem(observed, imply_ram_sym, loss_fiml, optimizer_obj) ############################################################################################ @testset "fiml_gradient" begin - @test test_gradient(model_ml, start_test_mean; atol = 1e-6) + test_gradient(model_ml, start_test_mean; atol = 1e-6) end @testset "fiml_gradient_symbolic" begin - @test test_gradient(model_ml_sym, start_test_mean; atol = 1e-6) + test_gradient(model_ml_sym, start_test_mean; atol = 1e-6) end ############################################################################################ diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index dd10f4be3..c74259aba 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -68,7 +68,7 @@ model_names = ["ml", "ml_cov", "ls_sym", "ridge", "constant", "ml_sym", "ml_weig for (model, name) in zip(models, model_names) try @testset "$(name)_gradient" begin - @test test_gradient(model, start_test; rtol = 1e-9) + test_gradient(model, start_test; rtol = 1e-9) end catch end @@ -242,7 +242,7 @@ model_names = ["ml", "ml_cov", "ls_sym", "ml_sym"] for (model, name) in zip(models, model_names) try @testset "$(name)_gradient_mean" begin - @test test_gradient(model, start_test_mean; rtol = 1e-9) + test_gradient(model, start_test_mean; rtol = 1e-9) end catch end @@ -322,11 +322,11 @@ model_ml_sym = Sem( ############################################################################################ @testset "fiml_gradient" begin - @test test_gradient(model_ml, start_test_mean; atol = 1e-6) + test_gradient(model_ml, start_test_mean; atol = 1e-6) end @testset "fiml_gradient_symbolic" begin - @test test_gradient(model_ml_sym, start_test_mean; atol = 1e-6) + test_gradient(model_ml_sym, start_test_mean; atol = 1e-6) end ############################################################################################ From 96aece8607a2b21f56f53a683571b4e8336c708d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 21:57:11 -0700 Subject: [PATCH 080/174] test_hessian(): do tests inside --- test/examples/helper.jl | 21 +++++++++---------- test/examples/political_democracy/by_parts.jl | 4 ++-- .../political_democracy/constructor.jl | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 155c357cb..a6d00b205 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -14,30 +14,29 @@ function test_gradient(model, parameters; rtol = 1e-10, atol = 0) end function test_hessian(model, parameters; rtol = 1e-4, atol = 0) - true_hessian = FiniteDiff.finite_difference_hessian(x -> objective!(model, x)[1], parameters) - hessian = zeros(size(true_hessian)); hessian .= 1.0 + true_hessian = FiniteDiff.finite_difference_hessian(Base.Fix1(objective!, model), parameters) + hessian = similar(parameters, size(true_hessian)) gradient = similar(parameters) # H + fill!(hessian, NaN) hessian!(hessian, model, parameters) - correct1 = isapprox(hessian, true_hessian; rtol = rtol, atol = atol) + @test hessian ≈ true_hessian rtol = rtol atol = atol # F and H - hessian .= 1.0 + fill!(hessian, NaN) objective_hessian!(hessian, model, parameters) - correct2 = isapprox(hessian, true_hessian; rtol = rtol, atol = atol) + @test hessian ≈ true_hessian rtol = rtol atol = atol # G and H - hessian .= 1.0 + fill!(hessian, NaN) gradient_hessian!(gradient, hessian, model, parameters) - correct3 = isapprox(hessian, true_hessian; rtol = rtol, atol = atol) + @test hessian ≈ true_hessian rtol = rtol atol = atol # F, G and H - hessian .= 1.0 + fill!(hessian, NaN) objective_gradient_hessian!(gradient, hessian, model, parameters) - correct4 = isapprox(hessian, true_hessian; rtol = rtol, atol = atol) - - return correct1 & correct2 & correct3 & correct4 + @test hessian ≈ true_hessian rtol = rtol atol = atol end fitmeasure_names_ml = Dict( diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 0894225e0..57eedb2b6 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -151,11 +151,11 @@ if semoptimizer == SemOptimizerOptim @testset "ml_hessians" begin - @test test_hessian(model_ml, start_test; atol = 1e-4) + test_hessian(model_ml, start_test; atol = 1e-4) end @testset "ls_hessians" begin - @test test_hessian(model_ls, start_test; atol = 1e-4) + test_hessian(model_ls, start_test; atol = 1e-4) end @testset "ml_solution_hessian" begin diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index c74259aba..ee192c057 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -170,11 +170,11 @@ if semoptimizer == SemOptimizerOptim ) @testset "ml_hessians" begin - @test test_hessian(model_ml, start_test; atol = 1e-4) + test_hessian(model_ml, start_test; atol = 1e-4) end @testset "ls_hessians" begin - @test test_hessian(model_ls, start_test; atol = 1e-4) + test_hessian(model_ls, start_test; atol = 1e-4) end @testset "ml_solution_hessian" begin From 397cf23ea9052bba23501271445bca99a4865ce1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 16:03:06 -0700 Subject: [PATCH 081/174] compare_estimates() -> test_estimates() * do tests inside * use param_values()/lavaan_param_values() --- test/examples/helper.jl | 221 +++--------------- test/examples/multigroup/build_models.jl | 18 +- test/examples/political_democracy/by_parts.jl | 22 +- .../political_democracy/constructor.jl | 22 +- test/unit_tests/sorting.jl | 2 +- 5 files changed, 64 insertions(+), 221 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index a6d00b205..89cd04a3b 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,3 +1,5 @@ +using LinearAlgebra: norm + function test_gradient(model, parameters; rtol = 1e-10, atol = 0) true_grad = FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), parameters) gradient = similar(parameters) @@ -58,10 +60,10 @@ fitmeasure_names_ls = Dict( ) function test_fitmeasures( - measures, - measures_lav; - rtol = 1e-4, - atol = 0, + measures, + measures_lav; + rtol = 1e-4, + atol = 0, fitmeasure_names = fitmeasure_names_ml) correct = [] for key in keys(fitmeasure_names) @@ -72,198 +74,39 @@ function test_fitmeasures( return correct end -function compare_estimates(partable::ParameterTable, partable_lav; - rtol = 1e-10, atol = 0, col = :estimate, lav_col = :est) - - correct = [] - - for i in findall(partable.columns[:free]) - - from = partable.columns[:from][i] - to = partable.columns[:to][i] - type = partable.columns[:parameter_type][i] - estimate = partable.columns[col][i] - - if from == Symbol("1") - - lav_ind = findall( - (partable_lav.lhs .== String(to)) .& - (partable_lav.op .== "~1")) - - if length(lav_ind) == 0 - throw(ErrorException("Parameter from: $from, to: $to, type: $type, could not be found in the lavaan solution")) - elseif length(lav_ind) > 1 - throw(ErrorException("At least one parameter was found twice in the lavaan solution")) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol) - push!(correct, is_correct) - end +function test_estimates(partable::ParameterTable, partable_lav; + rtol = 1e-10, atol = 0, col = :estimate, + lav_col = :est, lav_group = nothing, + skip::Bool = false) - else - - if type == :↔ - type = "~~" - elseif type == :→ - if (from ∈ partable.variables.latent) && (to ∈ partable.variables.observed) - type = "=~" - else - type = "~" - from, to = to, from - end - end - - if type == "~~" - - lav_ind = findall( - ( - ((partable_lav.lhs .== String(from)) .& (partable_lav.rhs .== String(to))) .| - ((partable_lav.lhs .== String(to)) .& (partable_lav.rhs .== String(from))) - ) .& - (partable_lav.op .== type) - ) - - if length(lav_ind) == 0 - throw(ErrorException("Parameter from: $from, to: $to, type: $type, could not be found in the lavaan solution")) - elseif length(lav_ind) > 1 - throw(ErrorException("At least one parameter was found twice in the lavaan solution")) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol) - push!(correct, is_correct) - end - - else - lav_ind = findall( - (partable_lav.lhs .== String(from)) .& - (partable_lav.rhs .== String(to)) .& - (partable_lav.op .== type)) - - if length(lav_ind) == 0 - throw(ErrorException("Parameter from: $from, to: $to, type: $type, could not be found in the lavaan solution")) - elseif length(lav_ind) > 1 - throw(ErrorException("At least one parameter was found twice in the lavaan solution")) - else - is_correct = isapprox(estimate, partable_lav[:, lav_col][lav_ind[1]]; rtol = rtol, atol = atol) - push!(correct, is_correct) - end - end - - end + actual = SEM.param_values(partable, col) + expected = SEM.lavaan_param_values(partable_lav, partable, lav_col, lav_group) + @test !any(isnan, actual) + @test !any(isnan, expected) + if skip # workaround skip=false not supported in earlier versions + @test actual ≈ expected rtol = rtol atol = atol norm=Base.Fix2(norm, Inf) skip = skip + else + @test actual ≈ expected rtol = rtol atol = atol norm=Base.Fix2(norm, Inf) end - - return all(correct) end -function compare_estimates(ens_partable::EnsembleParameterTable, partable_lav; +function test_estimates(ens_partable::EnsembleParameterTable, partable_lav; rtol = 1e-10, atol = 0, col = :estimate, lav_col = :est, - lav_groups) - - correct = [] - - for key in keys(ens_partable.tables) - - group = lav_groups[key] - partable = ens_partable.tables[key] - - for i in findall(partable.columns[:free]) - - from = partable.columns[:from][i] - to = partable.columns[:to][i] - type = partable.columns[:parameter_type][i] - estimate = partable.columns[col][i] - - if from == Symbol("1") - - lav_ind = findall( - (partable_lav.lhs .== String(to)) .& - (partable_lav.op .== "~1") .& - (partable_lav.group .== group)) - - if length(lav_ind) == 0 - throw(ErrorException("Mean parameter of variable $to could not be found in the lavaan solution")) - elseif length(lav_ind) > 1 - throw(ErrorException("At least one parameter was found twice in the lavaan solution")) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol) - push!(correct, is_correct) - end - - else - - if type == :↔ - type = "~~" - elseif type == :→ - if (from ∈ partable.variables.latent) && (to ∈ partable.variables.observed) - type = "=~" - else - type = "~" - from, to = to, from - end - end - - if type == "~~" - - lav_ind = findall( - ( - ((partable_lav.lhs .== String(from)) .& (partable_lav.rhs .== String(to))) .| - ((partable_lav.lhs .== String(to)) .& (partable_lav.rhs .== String(from))) - ) .& - (partable_lav.op .== type) .& - (partable_lav.group .== group) - ) - - if length(lav_ind) == 0 - throw(ErrorException("Parameter from: $from, to: $to, type: $type, could not be found in the lavaan solution")) - elseif length(lav_ind) > 1 - throw(ErrorException("At least one parameter was found twice in the lavaan solution")) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol) - push!(correct, is_correct) - end - - else - - lav_ind = findall( - (partable_lav.lhs .== String(from)) .& - (partable_lav.rhs .== String(to)) .& - (partable_lav.op .== type).& - (partable_lav.group .== group)) - - if length(lav_ind) == 0 - throw(ErrorException("Parameter $from $type $to could not be found in the lavaan solution")) - elseif length(lav_ind) > 1 - throw(ErrorException("At least one parameter was found twice in the lavaan solution")) - else - is_correct = isapprox( - estimate, - partable_lav[:, lav_col][lav_ind[1]]; - rtol = rtol, - atol = atol) - push!(correct, is_correct) - end - end - - end - - end + lav_groups::AbstractDict, skip::Bool = false) + actual = fill(NaN, nparams(ens_partable)) + expected = fill(NaN, nparams(ens_partable)) + for (key, partable) in pairs(ens_partable.tables) + SEM.param_values!(actual, partable, col) + SEM.lavaan_param_values!(expected, partable_lav, partable, lav_col, lav_groups[key]) end + @test !any(isnan, actual) + @test !any(isnan, expected) - return all(correct) + if skip # workaround skip=false not supported in earlier versions + @test actual ≈ expected rtol = rtol atol = atol norm=Base.Fix2(norm, Inf) skip = skip + else + @test actual ≈ expected rtol = rtol atol = atol norm=Base.Fix2(norm, Inf) + end end \ No newline at end of file diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index feab0f6b2..1e5785f97 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -26,7 +26,7 @@ end @testset "ml_solution_multigroup" begin solution = sem_fit(model_ml_multigroup) update_estimate!(partable, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) @@ -40,7 +40,7 @@ end update_partable!( partable, identifier(model_ml_multigroup), se_hessian(solution_ml), :se) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se, @@ -85,7 +85,7 @@ grad_fd = FiniteDiff.finite_difference_gradient(x -> SEM.objective!(model_ml_mul @testset "ml_solution_multigroup | sorted" begin solution = sem_fit(model_ml_multigroup) update_estimate!(partable_s, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) @@ -99,7 +99,7 @@ end update_partable!( partable_s, identifier(model_ml_multigroup), se_hessian(solution_ml), :se) - @test compare_estimates( + test_estimates( partable_s, solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se, @@ -154,7 +154,7 @@ end @testset "solution_user_defined_loss" begin solution = sem_fit(model_ml_multigroup) update_estimate!(partable, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) @@ -187,7 +187,7 @@ end @testset "ls_solution_multigroup" begin solution = sem_fit(model_ls_multigroup) update_estimate!(partable, solution) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) @@ -202,7 +202,7 @@ end update_partable!( partable, identifier(model_ls_multigroup), se_hessian(solution_ls), :se) - @test compare_estimates( + test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, col = :se, lav_col = :se, @@ -258,7 +258,7 @@ end @testset "fiml_solution_multigroup" begin solution = sem_fit(model_ml_multigroup) update_estimate!(partable_miss, solution) - @test compare_estimates( + test_estimates( partable_miss, solution_lav[:parameter_estimates_fiml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) @@ -272,7 +272,7 @@ end update_partable!( partable_miss, identifier(model_ml_multigroup), se_hessian(solution), :se) - @test compare_estimates( + test_estimates( partable_miss, solution_lav[:parameter_estimates_fiml]; atol = 1e-3, col = :se, lav_col = :se, diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 57eedb2b6..4810456ee 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -70,7 +70,7 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) @testset "$(name)_solution" begin solution = sem_fit(model) update_estimate!(partable, solution) - @test compare_estimates(partable, solution_lav[solution_name]; atol = 1e-2) + test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end catch end @@ -110,7 +110,7 @@ end atol = 1e-3)) update_partable!(partable, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ml]; + test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se) end @@ -122,7 +122,7 @@ end @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable, identifier(model_ls_sym), se_hessian(solution_ls), :se) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, + test_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, col = :se, lav_col = :se) end @@ -161,13 +161,13 @@ if semoptimizer == SemOptimizerOptim @testset "ml_solution_hessian" begin solution = sem_fit(model_ml) update_estimate!(partable, solution) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) + test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) end @testset "ls_solution_hessian" begin solution = sem_fit(model_ls) update_estimate!(partable, solution) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-3) skip=true + test_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-3, skip=true) end end @@ -235,7 +235,7 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) @testset "$(name)_solution_mean" begin solution = sem_fit(model) update_estimate!(partable_mean, solution) - @test compare_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) + test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end catch end @@ -251,7 +251,7 @@ end atol = 1e-3)) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; + test_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; atol = 0.002, col = :se, lav_col = :se) end @@ -265,7 +265,7 @@ end @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable_mean, identifier(model_ls), se_hessian(solution_ls), :se) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, col = :se, lav_col = :se) + test_estimates(partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, col = :se, lav_col = :se) end ############################################################################################ @@ -301,13 +301,13 @@ end @testset "fiml_solution" begin solution = sem_fit(model_ml) update_estimate!(partable_mean, solution) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin solution = sem_fit(model_ml_sym) update_estimate!(partable_mean, solution) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end ############################################################################################ @@ -320,6 +320,6 @@ end atol = 1e-3)) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-3, col = :se, lav_col = :se) end diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index ee192c057..154610ede 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -87,7 +87,7 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) @testset "$(name)_solution" begin solution = sem_fit(model) update_estimate!(partable, solution) - @test compare_estimates(partable, solution_lav[solution_name]; atol = 1e-2) + test_estimates(partable, solution_lav[solution_name]; atol = 1e-2) end catch end @@ -127,7 +127,7 @@ end atol = 1e-3)) update_partable!(partable, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ml]; + test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se) end @@ -139,7 +139,7 @@ end @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable, identifier(model_ls_sym), se_hessian(solution_ls), :se) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, + test_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, col = :se, lav_col = :se) end @@ -180,13 +180,13 @@ if semoptimizer == SemOptimizerOptim @testset "ml_solution_hessian" begin solution = sem_fit(model_ml) update_estimate!(partable, solution) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) + test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3) end @testset "ls_solution_hessian" begin solution = sem_fit(model_ls) update_estimate!(partable, solution) - @test compare_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 0.002, rtol = 0.0) skip=true + test_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 0.002, rtol = 0.0, skip=true) end end @@ -259,7 +259,7 @@ for (model, name, solution_name) in zip(models, model_names, solution_names) @testset "$(name)_solution_mean" begin solution = sem_fit(model) update_estimate!(partable_mean, solution) - @test compare_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) + test_estimates(partable_mean, solution_lav[solution_name]; atol = 1e-2) end catch end @@ -275,7 +275,7 @@ end atol = 0.002)) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; + test_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; atol = 0.002, col = :se, lav_col = :se) end @@ -289,7 +289,7 @@ end @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable_mean, identifier(model_ls), se_hessian(solution_ls), :se) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, col = :se, lav_col = :se) + test_estimates(partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, col = :se, lav_col = :se) end ############################################################################################ @@ -336,13 +336,13 @@ end @testset "fiml_solution" begin solution = sem_fit(model_ml) update_estimate!(partable_mean, solution) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end @testset "fiml_solution_symbolic" begin solution = sem_fit(model_ml_sym) update_estimate!(partable_mean, solution) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-2) end ############################################################################################ @@ -355,6 +355,6 @@ end atol = 1e-3)) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) - @test compare_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; + test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 0.002, col = :se, lav_col = :se) end diff --git a/test/unit_tests/sorting.jl b/test/unit_tests/sorting.jl index f3076b8ef..4e2c0fcb1 100644 --- a/test/unit_tests/sorting.jl +++ b/test/unit_tests/sorting.jl @@ -16,5 +16,5 @@ end @testset "ml_solution_sorted" begin solution_ml_sorted = sem_fit(model_ml_sorted) update_estimate!(partable, solution_ml_sorted) - @test SEM.compare_estimates(par_ml, partable, 0.01) + @test test_estimates(par_ml, partable, 0.01) end \ No newline at end of file From 352bf029383123eed335e5dbebb1d58197d6fc74 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 21:54:46 -0700 Subject: [PATCH 082/174] comp_fitmeasures() -> test_fitmeasures() --- test/examples/helper.jl | 10 ++++---- test/examples/multigroup/build_models.jl | 16 ++++++------- test/examples/political_democracy/by_parts.jl | 24 +++++++++---------- .../political_democracy/constructor.jl | 20 ++++++++-------- 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 89cd04a3b..797490f35 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -65,13 +65,11 @@ function test_fitmeasures( rtol = 1e-4, atol = 0, fitmeasure_names = fitmeasure_names_ml) - correct = [] - for key in keys(fitmeasure_names) - measure = measures[key] - measure_lav = measures_lav.x[measures_lav[:, 1] .== fitmeasure_names[key]][1] - push!(correct, isapprox(measure, measure_lav; rtol = rtol, atol = atol)) + + @testset "$name" for (key, name) in pairs(fitmeasure_names) + measure_lav = measures_lav.x[findfirst(==(name), measures_lav[!, 1])] + @test measures[key] ≈ measure_lav rtol = rtol atol = atol end - return correct end function test_estimates(partable::ParameterTable, partable_lav; diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 1e5785f97..1ac21521a 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -34,9 +34,9 @@ end @testset "fitmeasures/se_ml" begin solution_ml = sem_fit(model_ml_multigroup) - @test all(test_fitmeasures( + test_fitmeasures( fit_measures(solution_ml), - solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7)) + solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7) update_partable!( partable, identifier(model_ml_multigroup), se_hessian(solution_ml), :se) @@ -93,9 +93,9 @@ end @testset "fitmeasures/se_ml | sorted" begin solution_ml = sem_fit(model_ml_multigroup) - @test all(test_fitmeasures( + test_fitmeasures( fit_measures(solution_ml), - solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7)) + solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7) update_partable!( partable_s, identifier(model_ml_multigroup), se_hessian(solution_ml), :se) @@ -195,10 +195,10 @@ end @testset "fitmeasures/se_ls" begin solution_ls = sem_fit(model_ls_multigroup) - @test all(test_fitmeasures( + test_fitmeasures( fit_measures(solution_ls), solution_lav[:fitmeasures_ls]; - fitmeasure_names = fitmeasure_names_ls, rtol = 1e-2, atol = 1e-5)) + fitmeasure_names = fitmeasure_names_ls, rtol = 1e-2, atol = 1e-5) update_partable!( partable, identifier(model_ls_multigroup), se_hessian(solution_ls), :se) @@ -266,9 +266,9 @@ end @testset "fitmeasures/se_fiml" begin solution = sem_fit(model_ml_multigroup) - @test all(test_fitmeasures( + test_fitmeasures( fit_measures(solution), - solution_lav[:fitmeasures_fiml]; rtol = 1e-3, atol = 0)) + solution_lav[:fitmeasures_fiml]; rtol = 1e-3, atol = 0) update_partable!( partable_miss, identifier(model_ml_multigroup), se_hessian(solution), :se) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 4810456ee..9b074fe6f 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -106,8 +106,8 @@ end @testset "fitmeasures/se_ml" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; - atol = 1e-3)) + test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; + atol = 1e-3) update_partable!(partable, identifier(model_ml), se_hessian(solution_ml), :se) test_estimates(partable, solution_lav[:parameter_estimates_ml]; @@ -117,8 +117,8 @@ end @testset "fitmeasures/se_ls" begin solution_ls = sem_fit(model_ls_sym) fm = fit_measures(solution_ls) - @test all(test_fitmeasures(fm, solution_lav[:fitmeasures_ls]; atol = 1e-3, - fitmeasure_names = fitmeasure_names_ls)) + test_fitmeasures(fm, solution_lav[:fitmeasures_ls]; atol = 1e-3, + fitmeasure_names = fitmeasure_names_ls) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable, identifier(model_ls_sym), se_hessian(solution_ls), :se) @@ -247,8 +247,8 @@ end @testset "fitmeasures/se_ml_mean" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; - atol = 1e-3)) + test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; + atol = 1e-3) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) test_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; @@ -258,10 +258,10 @@ end @testset "fitmeasures/se_ls_mean" begin solution_ls = sem_fit(model_ls) fm = fit_measures(solution_ls) - @test all(test_fitmeasures(fm, - solution_lav[:fitmeasures_ls_mean]; - atol = 1e-3, - fitmeasure_names = fitmeasure_names_ls)) + test_fitmeasures(fm, + solution_lav[:fitmeasures_ls_mean]; + atol = 1e-3, + fitmeasure_names = fitmeasure_names_ls) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable_mean, identifier(model_ls), se_hessian(solution_ls), :se) @@ -316,8 +316,8 @@ end @testset "fitmeasures/se_fiml" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; - atol = 1e-3)) + test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; + atol = 1e-3) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 154610ede..0ca0084ad 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -123,8 +123,8 @@ end @testset "fitmeasures/se_ml" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; - atol = 1e-3)) + test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; + atol = 1e-3) update_partable!(partable, identifier(model_ml), se_hessian(solution_ml), :se) test_estimates(partable, solution_lav[:parameter_estimates_ml]; @@ -134,8 +134,8 @@ end @testset "fitmeasures/se_ls" begin solution_ls = sem_fit(model_ls_sym) fm = fit_measures(solution_ls) - @test all(test_fitmeasures(fm, solution_lav[:fitmeasures_ls]; atol = 1e-3, - fitmeasure_names = fitmeasure_names_ls)) + test_fitmeasures(fm, solution_lav[:fitmeasures_ls]; atol = 1e-3, + fitmeasure_names = fitmeasure_names_ls) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable, identifier(model_ls_sym), se_hessian(solution_ls), :se) @@ -271,8 +271,8 @@ end @testset "fitmeasures/se_ml_mean" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; - atol = 0.002)) + test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; + atol = 0.002) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) test_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; @@ -282,10 +282,10 @@ end @testset "fitmeasures/se_ls_mean" begin solution_ls = sem_fit(model_ls) fm = fit_measures(solution_ls) - @test all(test_fitmeasures(fm, + test_fitmeasures(fm, solution_lav[:fitmeasures_ls_mean]; atol = 1e-3, - fitmeasure_names = fitmeasure_names_ls)) + fitmeasure_names = fitmeasure_names_ls) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) update_partable!(partable_mean, identifier(model_ls), se_hessian(solution_ls), :se) @@ -351,8 +351,8 @@ end @testset "fitmeasures/se_fiml" begin solution_ml = sem_fit(model_ml) - @test all(test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; - atol = 1e-3)) + test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; + atol = 1e-3) update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; From 06df8035238f53b9b1a5726cdc0daa902e4faf85 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 21:57:11 -0700 Subject: [PATCH 083/174] tests: tiny improvements --- test/examples/political_democracy/by_parts.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 9b074fe6f..84a0a118e 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -80,7 +80,7 @@ end solution_ridge = sem_fit(model_ridge) solution_ml = sem_fit(model_ml) # solution_ridge_id = sem_fit(model_ridge_id) - @test abs(solution_ridge.minimum - solution_ml.minimum) < 1 + @test solution_ridge.minimum < solution_ml.minimum + 1 end # test constant objective value @@ -95,9 +95,9 @@ end @testset "ml_solution_weighted" begin solution_ml = sem_fit(model_ml) solution_ml_weighted = sem_fit(model_ml_weighted) - @test isapprox(solution(solution_ml), solution(solution_ml_weighted), rtol = 1e-3) - @test isapprox(n_obs(model_ml)*StructuralEquationModels.minimum(solution_ml), - StructuralEquationModels.minimum(solution_ml_weighted), rtol = 1e-6) + @test solution(solution_ml) ≈ solution(solution_ml_weighted) rtol = 1e-3 + @test n_obs(model_ml)*StructuralEquationModels.minimum(solution_ml) ≈ + StructuralEquationModels.minimum(solution_ml_weighted) rtol = 1e-6 end ############################################################################################ From f52f4d5647a1abee5cf4d48f4f6b6b9126166a8d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 15 Mar 2024 08:36:35 -0700 Subject: [PATCH 084/174] tests: use approx op --- test/examples/recover_parameters/recover_parameters_twofact.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 18c9aef07..115a06721 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -64,4 +64,4 @@ model_ml = Sem(semobserved, imply_ml, loss_ml, optimizer) objective!(model_ml, true_val) solution_ml = sem_fit(model_ml) -@test isapprox(true_val, solution(solution_ml); atol = .05) \ No newline at end of file +@test true_val ≈ solution(solution_ml) atol = .05 \ No newline at end of file From 5a12f62c51beea07a88ef059ecfbe21d5816d42b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 15 Mar 2024 08:37:26 -0700 Subject: [PATCH 085/174] tests: fix ensembl ctor --- test/examples/multigroup/multigroup.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index 60345dace..de0ce7c55 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -90,8 +90,7 @@ graph = @StenoGraph begin _(latent_vars) ⇔ _(latent_vars) end -partable = EnsembleParameterTable(; - graph = graph, +partable = EnsembleParameterTable(graph; observed_vars = observed_vars, latent_vars = latent_vars, groups = [:Pasteur, :Grant_White]) @@ -118,8 +117,7 @@ graph = @StenoGraph begin Symbol("1") → _(observed_vars) end -partable_miss = EnsembleParameterTable(; - graph = graph, +partable_miss = EnsembleParameterTable(graph; observed_vars = observed_vars, latent_vars = latent_vars, groups = [:Pasteur, :Grant_White]) From cc3ae119e9e25ec57f847f38fb385d45eb70912e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 15 Mar 2024 12:15:47 -0700 Subject: [PATCH 086/174] tests: relax multithreading check --- test/unit_tests/multithreading.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit_tests/multithreading.jl b/test/unit_tests/multithreading.jl index ba8770f19..690bac270 100644 --- a/test/unit_tests/multithreading.jl +++ b/test/unit_tests/multithreading.jl @@ -2,6 +2,6 @@ using Test if haskey(ENV, "JULIA_ON_CI") @testset "multithreading_enabled" begin - @test Threads.nthreads() == 8 + @test Threads.nthreads() >= 8 end end \ No newline at end of file From f621f207f464b00721689e2d4154efb1ca6e4f53 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:42:47 -0700 Subject: [PATCH 087/174] tests: use ismissing() --- test/examples/political_democracy/constructor.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 0ca0084ad..3d43a6818 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -136,7 +136,7 @@ end fm = fit_measures(solution_ls) test_fitmeasures(fm, solution_lav[:fitmeasures_ls]; atol = 1e-3, fitmeasure_names = fitmeasure_names_ls) - @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) + @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) update_partable!(partable, identifier(model_ls_sym), se_hessian(solution_ls), :se) test_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, @@ -286,7 +286,7 @@ end solution_lav[:fitmeasures_ls_mean]; atol = 1e-3, fitmeasure_names = fitmeasure_names_ls) - @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) + @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) update_partable!(partable_mean, identifier(model_ls), se_hessian(solution_ls), :se) test_estimates(partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, col = :se, lav_col = :se) From 137bf0cba09f0ee7008e80238639079497d5f6ff Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 17:21:14 -0700 Subject: [PATCH 088/174] tests: SEM module alias --- test/examples/multigroup/multigroup.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index de0ce7c55..b551415c4 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -1,5 +1,8 @@ using StructuralEquationModels, Test, FiniteDiff using LinearAlgebra: diagind, LowerTriangular + +SEM = StructuralEquationModels + # using StructuralEquationModels as SEM include( joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), From 9be661531f8fb886f9221bdf641e6dc788287834 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 20:37:36 -0700 Subject: [PATCH 089/174] tests: use update_se_hessian!() --- src/StructuralEquationModels.jl | 2 +- test/examples/multigroup/build_models.jl | 12 ++++------- test/examples/political_democracy/by_parts.jl | 20 +++++++++---------- .../political_democracy/constructor.jl | 10 +++++----- test/unit_tests/bootstrap.jl | 3 +-- 5 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index ee7b4172f..39651885c 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -103,7 +103,7 @@ export AbstractSem, objective!, gradient!, hessian!, objective_gradient!, objective_hessian!, gradient_hessian!, objective_gradient_hessian!, ParameterTable, - EnsembleParameterTable, update_partable!, update_estimate!, update_start!, + EnsembleParameterTable, update_partable!, update_estimate!, update_start!, update_se_hessian!, Fixed, fixed, Start, start, Label, label, sort_vars!, sort_vars, get_identifier_indices, RAMMatrices, diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 1ac21521a..7dbab094d 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -38,8 +38,7 @@ end fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7) - update_partable!( - partable, identifier(model_ml_multigroup), se_hessian(solution_ml), :se) + update_se_hessian!(partable, solution_ml) test_estimates( partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, @@ -97,8 +96,7 @@ end fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; rtol = 1e-2, atol = 1e-7) - update_partable!( - partable_s, identifier(model_ml_multigroup), se_hessian(solution_ml), :se) + update_se_hessian!(partable_s, solution_ml) test_estimates( partable_s, solution_lav[:parameter_estimates_ml]; atol = 1e-3, @@ -200,8 +198,7 @@ end solution_lav[:fitmeasures_ls]; fitmeasure_names = fitmeasure_names_ls, rtol = 1e-2, atol = 1e-5) - update_partable!( - partable, identifier(model_ls_multigroup), se_hessian(solution_ls), :se) + update_se_hessian!(partable, solution_ls) test_estimates( partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, @@ -270,8 +267,7 @@ end fit_measures(solution), solution_lav[:fitmeasures_fiml]; rtol = 1e-3, atol = 0) - update_partable!( - partable_miss, identifier(model_ml_multigroup), se_hessian(solution), :se) + update_se_hessian!(partable_miss, solution) test_estimates( partable_miss, solution_lav[:parameter_estimates_fiml]; atol = 1e-3, diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 84a0a118e..fd56ab68c 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -109,7 +109,7 @@ end test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) - update_partable!(partable, identifier(model_ml), se_hessian(solution_ml), :se) + update_se_hessian!(partable, solution_ml) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se) end @@ -121,7 +121,7 @@ end fitmeasure_names = fitmeasure_names_ls) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) - update_partable!(partable, identifier(model_ls_sym), se_hessian(solution_ls), :se) + update_se_hessian!(partable, solution_ls) test_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, col = :se, lav_col = :se) end @@ -135,13 +135,13 @@ if semoptimizer == SemOptimizerOptim optimizer_obj = SemOptimizerOptim( algorithm = Newton( - ;linesearch = BackTracking(order=3), + ;linesearch = BackTracking(order=3), alphaguess = InitialHagerZhang() ) ) imply_sym_hessian_vech = RAMSymbolic(specification = spec, vech = true, hessian = true) - + imply_sym_hessian = RAMSymbolic(specification = spec, hessian = true) @@ -201,9 +201,9 @@ optimizer_obj = semoptimizer() model_ml = Sem(observed, imply_ram, loss_ml, optimizer_obj) model_ls = Sem( - observed, - RAMSymbolic(specification = spec_mean, meanstructure = true, vech = true), - loss_wls, + observed, + RAMSymbolic(specification = spec_mean, meanstructure = true, vech = true), + loss_wls, optimizer_obj) model_ml_sym = Sem(observed, imply_ram_sym, loss_ml, optimizer_obj) @@ -250,7 +250,7 @@ end test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; atol = 1e-3) - update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) + update_se_hessian!(partable_mean, solution_ml) test_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; atol = 0.002, col = :se, lav_col = :se) end @@ -264,7 +264,7 @@ end fitmeasure_names = fitmeasure_names_ls) @test (fm[:AIC] === missing) & (fm[:BIC] === missing) & (fm[:minus2ll] === missing) - update_partable!(partable_mean, identifier(model_ls), se_hessian(solution_ls), :se) + update_se_hessian!(partable_mean, solution_ls) test_estimates(partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, col = :se, lav_col = :se) end @@ -319,7 +319,7 @@ end test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; atol = 1e-3) - update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) + update_se_hessian!(partable_mean, solution_ml) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 1e-3, col = :se, lav_col = :se) end diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 3d43a6818..8dfa86a46 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -126,7 +126,7 @@ end test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml]; atol = 1e-3) - update_partable!(partable, identifier(model_ml), se_hessian(solution_ml), :se) + update_se_hessian!(partable, solution_ml) test_estimates(partable, solution_lav[:parameter_estimates_ml]; atol = 1e-3, col = :se, lav_col = :se) end @@ -138,7 +138,7 @@ end fitmeasure_names = fitmeasure_names_ls) @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) - update_partable!(partable, identifier(model_ls_sym), se_hessian(solution_ls), :se) + update_se_hessian!(partable, solution_ls) test_estimates(partable, solution_lav[:parameter_estimates_ls]; atol = 1e-2, col = :se, lav_col = :se) end @@ -274,7 +274,7 @@ end test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_ml_mean]; atol = 0.002) - update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) + update_se_hessian!(partable_mean, solution_ml) test_estimates(partable_mean, solution_lav[:parameter_estimates_ml_mean]; atol = 0.002, col = :se, lav_col = :se) end @@ -288,7 +288,7 @@ end fitmeasure_names = fitmeasure_names_ls) @test ismissing(fm[:AIC]) && ismissing(fm[:BIC]) && ismissing(fm[:minus2ll]) - update_partable!(partable_mean, identifier(model_ls), se_hessian(solution_ls), :se) + update_se_hessian!(partable_mean, solution_ls) test_estimates(partable_mean, solution_lav[:parameter_estimates_ls_mean]; atol = 1e-2, col = :se, lav_col = :se) end @@ -354,7 +354,7 @@ end test_fitmeasures(fit_measures(solution_ml), solution_lav[:fitmeasures_fiml]; atol = 1e-3) - update_partable!(partable_mean, identifier(model_ml), se_hessian(solution_ml), :se) + update_se_hessian!(partable_mean, solution_ml) test_estimates(partable_mean, solution_lav[:parameter_estimates_fiml]; atol = 0.002, col = :se, lav_col = :se) end diff --git a/test/unit_tests/bootstrap.jl b/test/unit_tests/bootstrap.jl index e519e5f50..5be44c67a 100644 --- a/test/unit_tests/bootstrap.jl +++ b/test/unit_tests/bootstrap.jl @@ -1,6 +1,5 @@ solution_ml = sem_fit(model_ml) bs = se_bootstrap(solution_ml; n_boot = 20) -se = se_hessian(solution_ml) -update_partable!(partable, solution_ml, se, :se) +update_se_hessian!(partable, solution_ml) update_partable!(partable, solution_ml, bs, :se_boot) \ No newline at end of file From 287a3ef0e006bba86de9a67351fdce186c3340ac Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 16 Mar 2024 12:47:20 -0700 Subject: [PATCH 090/174] remove module spec --- src/diff/Empty.jl | 2 +- src/imply/empty.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diff/Empty.jl b/src/diff/Empty.jl index cce869253..0780907e0 100644 --- a/src/diff/Empty.jl +++ b/src/diff/Empty.jl @@ -34,5 +34,5 @@ update_observed(optimizer::SemOptimizerEmpty, observed::SemObserved; kwargs...) ############################################################################################ function Base.show(io::IO, struct_inst::SemOptimizerEmpty) - StructuralEquationModels.print_type_name(io, struct_inst) + print_type_name(io, struct_inst) end \ No newline at end of file diff --git a/src/imply/empty.jl b/src/imply/empty.jl index ce4664a4a..3e77bef34 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -39,7 +39,7 @@ function ImplyEmpty(; kwargs...) ram_matrices = RAMMatrices(specification) - identifier = StructuralEquationModels.identifier(ram_matrices) + identifier = identifier(ram_matrices) n_par = length(ram_matrices.parameters) From 6ec5540b338b7d05405aa23397f948cc3eab1dce Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:09:15 -0700 Subject: [PATCH 091/174] remove no-op method --- src/observed/covariance.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index b2d58e949..332b8fbf0 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -87,7 +87,7 @@ function SemObservedCovariance(; if !isnothing(spec_colnames) obs_cov = reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) - obs_mean = reorder_obs_mean(obs_mean, spec_colnames, obs_colnames) + isnothing(obs_mean) || (obs_mean = reorder_obs_mean(obs_mean, spec_colnames, obs_colnames)) end n_man = Float64(size(obs_cov, 1)) @@ -125,7 +125,6 @@ function reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) end # reorder means ---------------------------------------------------------------------------- -reorder_obs_mean(obs_mean::Nothing, spec_colnames, obs_colnames) = nothing function reorder_obs_mean(obs_mean, spec_colnames, obs_colnames) if spec_colnames == obs_colnames From d8dc844db87121c0eda86f9a2ff07f11390501b7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:17:13 -0700 Subject: [PATCH 092/174] start_fabin3: check mean data and model --- src/additional_functions/start_val/start_fabin3.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index d30698bf9..c29e17727 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -49,7 +49,9 @@ function start_fabin3( end -function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) +function start_fabin3(ram_matrices::RAMMatrices, + Σ::AbstractMatrix, + μ::Union{AbstractVector, Nothing}) A, S, F, M, n_par = ram_matrices.A, @@ -58,6 +60,12 @@ function start_fabin3(ram_matrices::RAMMatrices, Σ, μ) ram_matrices.M, nparams(ram_matrices) + if !isnothing(M) && isnothing(μ) + throw(ArgumentError("RAM has meanstructure, but no observed means provided.")) + elseif isnothing(M) && !isnothing(μ) + throw(ArgumentError("RAM has no meanstructure, but observed means provided.")) + end + start_val = zeros(n_par) F_var2obs = Dict(i => F.rowval[F.colptr[i]] for i in axes(F, 2) if isobserved_var(ram_matrices, i)) From 76b35dd540741d20876db51d13d059203760006b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 23:46:42 -0700 Subject: [PATCH 093/174] identifier/parameters -> params --- src/StructuralEquationModels.jl | 7 +- src/additional_functions/identifier.jl | 24 ++---- .../start_val/start_partable.jl | 19 +++-- src/frontend/fit/SemFit.jl | 2 + src/frontend/fit/fitmeasures/n_par.jl | 14 +--- .../specification/EnsembleParameterTable.jl | 47 ++++++----- src/frontend/specification/ParameterTable.jl | 59 +++++++++++--- src/frontend/specification/RAMMatrices.jl | 81 +++++++++---------- src/frontend/specification/StenoGraphs.jl | 25 +++++- src/imply/RAM/generic.jl | 21 ++--- src/imply/RAM/symbolic.jl | 8 +- src/imply/empty.jl | 19 ++--- src/loss/regularization/ridge.jl | 3 +- src/types.jl | 21 +++-- test/examples/helper.jl | 2 + test/examples/multigroup/build_models.jl | 2 + test/examples/multigroup/multigroup.jl | 4 +- .../political_democracy/constructor.jl | 1 + .../political_democracy.jl | 14 +++- .../recover_parameters_twofact.jl | 2 +- 20 files changed, 208 insertions(+), 167 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 39651885c..baf890737 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -60,7 +60,7 @@ include("additional_functions/start_val/start_simple.jl") include("additional_functions/artifacts.jl") include("additional_functions/simulation.jl") # identifier -include("additional_functions/identifier.jl") +#include("additional_functions/identifier.jl") # fit measures include("frontend/fit/fitmeasures/AIC.jl") include("frontend/fit/fitmeasures/BIC.jl") @@ -105,10 +105,9 @@ export AbstractSem, ParameterTable, EnsembleParameterTable, update_partable!, update_estimate!, update_start!, update_se_hessian!, Fixed, fixed, Start, start, Label, label, sort_vars!, sort_vars, - get_identifier_indices, - RAMMatrices, + RAMMatrices, RAMMatrices!, - identifier, nparams, + params, nparams, fit_measures, AIC, BIC, χ², df, fit_measures, minus2ll, n_obs, p_value, RMSEA, n_man, EmMVNModel, diff --git a/src/additional_functions/identifier.jl b/src/additional_functions/identifier.jl index 48773adcc..2d2bdb694 100644 --- a/src/additional_functions/identifier.jl +++ b/src/additional_functions/identifier.jl @@ -2,37 +2,23 @@ # get parameter identifier ############################################################################################ -identifier(sem_fit::SemFit) = identifier(sem_fit.model) -identifier(model::AbstractSemSingle) = identifier(model.imply) -identifier(model::SemEnsemble) = model.identifier - -############################################################################################ -# construct identifier -############################################################################################ - -identifier(ram_matrices::RAMMatrices) = - Dict{Symbol, Int64}(ram_matrices.parameters .=> 1:length(ram_matrices.parameters)) -function identifier(partable::ParameterTable) - _, _, identifier = get_par_npar_identifier(partable) - return identifier -end ############################################################################################ # get indices of a Vector of parameter labels ############################################################################################ -get_identifier_indices(parameters, identifier::Dict{Symbol, Int}) = +get_identifier_indices(parameters, identifier::Dict{Symbol, Int}) = [identifier[par] for par in parameters] -get_identifier_indices(parameters, obj::Union{SemFit, AbstractSemSingle, SemEnsemble, SemImply}) = - get_identifier_indices(parameters, identifier(obj)) +get_identifier_indices(parameters, obj::Union{SemFit, AbstractSemSingle, SemEnsemble, SemImply}) = + get_identifier_indices(parameters, params(obj)) function get_identifier_indices(parameters, obj::Union{ParameterTable, RAMMatrices}) @warn "You are trying to find parameter indices from a ParameterTable or RAMMatrices object. \n If your model contains user-defined types, this may lead to wrong results. \n - To be on the safe side, try to reference parameters by labels or query the indices from + To be on the safe side, try to reference parameters by labels or query the indices from the constructed model (`get_identifier_indices(parameters, model)`)." maxlog=1 - return get_identifier_indices(parameters, identifier(obj)) + return get_identifier_indices(parameters, params(obj)) end ############################################################################################ diff --git a/src/additional_functions/start_val/start_partable.jl b/src/additional_functions/start_val/start_partable.jl index 78ac4eed9..038634f2b 100644 --- a/src/additional_functions/start_val/start_partable.jl +++ b/src/additional_functions/start_val/start_partable.jl @@ -22,20 +22,19 @@ function start_parameter_table(observed, imply, optimizer, args...; kwargs...) kwargs...) end -function start_parameter_table(ram_matrices::RAMMatrices; parameter_table::ParameterTable, kwargs...) +function start_parameter_table(ram::RAMMatrices; partable::ParameterTable, kwargs...) start_val = zeros(0) - for identifier_ram in ram_matrices.parameters - found = false - for (i, identifier_table) in enumerate(parameter_table.identifier) - if identifier_ram == identifier_table - push!(start_val, parameter_table.start[i]) - found = true - break - end + param_indices = Dict(param => i for (i, param) in enumerate(params(ram))) + + for (i, param) in enumerate(partable.columns.identifier) + par_ind = get(param_indices, param, nothing) + if !isnothing(par_ind) + isfinite(partable.start[i]) && (start_val[i] = partable.start[i]) + else + throw(ErrorException("Parameter $(param) is not in the parameter table.")) end - if !found throw(ErrorException("At least one parameter could not be found in the parameter table.")) end end return start_val diff --git a/src/frontend/fit/SemFit.jl b/src/frontend/fit/SemFit.jl index 59dc0ccfc..6671b1eab 100644 --- a/src/frontend/fit/SemFit.jl +++ b/src/frontend/fit/SemFit.jl @@ -25,6 +25,8 @@ mutable struct SemFit{Mi, So, St, Mo, O} optimization_result::O end +params(fit::SemFit) = params(fit.model) + ############################################################################################ # pretty printing ############################################################################################ diff --git a/src/frontend/fit/fitmeasures/n_par.jl b/src/frontend/fit/fitmeasures/n_par.jl index c569892a3..5a60dd7c2 100644 --- a/src/frontend/fit/fitmeasures/n_par.jl +++ b/src/frontend/fit/fitmeasures/n_par.jl @@ -3,18 +3,8 @@ ############################################################################################ """ nparams(sem_fit::SemFit) - nparams(model::AbstractSemSingle) - nparams(model::SemEnsemble) - nparams(identifier::Dict) + nparams(model::AbstractSem) Return the number of parameters. """ -function nparams end - -nparams(fit::SemFit) = nparams(fit.model) - -nparams(model::AbstractSemSingle) = nparams(model.imply) - -nparams(model::SemEnsemble) = nparams(model.identifier) - -nparams(identifier::Dict) = length(identifier) +nparams(obj::Any) = length(params(obj)) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index 8295ff948..ef8b932ab 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -2,8 +2,9 @@ ### Types ############################################################################################ -mutable struct EnsembleParameterTable{C <: AbstractDict{<:Any, ParameterTable}} <: AbstractParameterTable - tables::C +struct EnsembleParameterTable{K} <: AbstractParameterTable + tables::Dict{K, ParameterTable} + params::Vector{Symbol} end ############################################################################################ @@ -11,9 +12,26 @@ end ############################################################################################ # constuct an empty table -function EnsembleParameterTable(::Nothing) - tables = Dict{Symbol, ParameterTable}() - return EnsembleParameterTable(tables) +EnsembleParameterTable(::Nothing; + params::Union{Nothing, Vector{Symbol}} = nothing) = + EnsembleParameterTable{Symbol}( + Dict{Symbol, ParameterTable}(), + isnothing(params) ? Symbol[] : copy(params)) + +# dictionary of SEM specifications +function EnsembleParameterTable(spec_ensemble::AbstractDict{K, V}; + params::Union{Nothing, Vector{Symbol}} = nothing) where {K, V <: SemSpecification} + partables = Dict{K, ParameterTable}( + group => convert(ParameterTable, spec; params = params) + for (group, spec) in pairs(spec_ensemble)) + + # collect all SEM parameters in ensemble if not specified + params = isnothing(params) ? + unique(mapreduce(SEM.params, vcat, + values(partables), init=Vector{Symbol}())) : + copy(params) + + return EnsembleParameterTable{K}(partables, params) end ############################################################################################ @@ -44,15 +62,6 @@ end return DataFrame(out) end =# -############################################################################################ -### get parameter table from RAMMatrices -############################################################################################ - -EnsembleParameterTable(spec_ensemble::AbstractDict{K}) where K = - EnsembleParameterTable(Dict{K, ParameterTable}( - group => convert(ParameterTable, spec) - for (group, spec) in pairs(spec_ensemble))) - ############################################################################################ ### Pretty Printing ############################################################################################ @@ -100,9 +109,11 @@ Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] ############################################################################################ # update generic --------------------------------------------------------------------------- -function update_partable!(partable::EnsembleParameterTable, model_identifier::AbstractDict, vec, column) - for k in keys(partable.tables) - update_partable!(partable.tables[k], model_identifier, vec, column) +function update_partable!(partables::EnsembleParameterTable, + params::AbstractVector, param_values::AbstractVector, + column::Symbol) + for partable in values(partables.tables) + update_partable!(partable, params, param_values, column) end - return partable + return partables end \ No newline at end of file diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 07ea2c630..ab177f7ed 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -5,6 +5,7 @@ mutable struct ParameterTable{C, V} <: AbstractParameterTable columns::C variables::V + params::Vector{Symbol} end ############################################################################################ @@ -13,7 +14,8 @@ end # constuct an empty table function ParameterTable(; observed_vars::Union{AbstractVector{Symbol}, Nothing}=nothing, - latent_vars::Union{AbstractVector{Symbol}, Nothing}=nothing) + latent_vars::Union{AbstractVector{Symbol}, Nothing}=nothing, + params::Union{AbstractVector{Symbol}, Nothing}=nothing) columns = ( from = Vector{Symbol}(), parameter_type = Vector{Symbol}(), @@ -23,17 +25,42 @@ function ParameterTable(; observed_vars::Union{AbstractVector{Symbol}, Nothing}= start = Vector{Float64}(), estimate = Vector{Float64}(), se = Vector{Float64}(), - identifier = Vector{Symbol}(), + param = Vector{Symbol}(), ) - variables = ( + vars = ( latent = !isnothing(latent_vars) ? copy(latent_vars) : Vector{Symbol}(), observed = !isnothing(observed_vars) ? copy(observed_vars) : Vector{Symbol}(), sorted = Vector{Symbol}() ) - return ParameterTable(columns, variables) + return ParameterTable(columns, vars, + !isnothing(params) ? copy(params) : Vector{Symbol}()) end + +function check_params(params::AbstractVector{Symbol}, partable_ids::AbstractVector{Symbol}) + all_refs = Set(id for id in partable_ids if id != :const) + undecl_params = setdiff(all_refs, params) + if !isempty(undecl_params) + throw(ArgumentError("The following $(length(undecl_params)) parameters present in the table, but are not declared: " * + join(sort!(collect(undecl_params))))) + end +end + +# new parameter table with different parameters order +function ParameterTable(partable::ParameterTable; + params::Union{AbstractVector{Symbol}, Nothing}=nothing) + isnothing(params) || check_params(params, partable.columns.param) + + newtable = ParameterTable(observed_vars = observed_vars(partable), + latent_vars = latent_vars(partable), + params = params) + newtable.columns = NamedTuple(col => copy(values) + for (col, values) in pairs(partable.columns)) + + return newtable +end + vars(partable::ParameterTable) = !isempty(partable.variables.sorted) ? partable.variables.sorted : vcat(partable.variables.observed, partable.variables.latent) @@ -51,6 +78,12 @@ function Base.convert(::Type{Dict}, partable::ParameterTable) return partable.columns end +function Base.convert(::Type{ParameterTable}, partable::ParameterTable; + params::Union{AbstractVector{Symbol}, Nothing}=nothing) + return isnothing(params) || partable.params == params ? partable : + ParameterTable(partable; params=params) +end + function DataFrames.DataFrame( partable::ParameterTable; columns::Union{AbstractVector{Symbol}, Nothing} = nothing) @@ -214,17 +247,19 @@ end # update generic --------------------------------------------------------------------------- function update_partable!(partable::ParameterTable, - model_identifier::AbstractDict, + params::AbstractVector{Symbol}, values::AbstractVector, column::Symbol) + length(params) == length(values) || + throw(ArgumentError("The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same")) coldata = partable.columns[column] + fixed_values = partable.columns.value_fixed + param_index = Dict(zip(params, eachindex(params))) resize!(coldata, length(partable)) for (i, id) in enumerate(partable.columns.identifier) - if !(id == :const) - coldata[i] = values[model_identifier[id]] - elseif id == :const - coldata[i] = zero(eltype(values)) - end + coldata[i] = id != :const ? + values[param_index[id]] : + fixed_values[i] end return partable end @@ -239,7 +274,7 @@ Write `vec` to `column` of `partable`. """ update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, values::AbstractVector, column::Symbol) = - update_partable!(partable, identifier(sem_fit), values, column) + update_partable!(partable, params(sem_fit), values, column) # update estimates ------------------------------------------------------------------------- """ @@ -275,7 +310,7 @@ function update_start!( if !(start_val isa Vector) start_val = start_val(model; kwargs...) end - return update_partable!(partable, identifier(model), start_val, :start) + return update_partable!(partable, params(model), start_val, :start) end # update partable standard errors ---------------------------------------------------------- diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 0a611b54c..95de6361c 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -8,7 +8,7 @@ struct RAMMatrices <: SemSpecification S::ParamsMatrix{Float64} F::SparseMatrixCSC{Float64} M::Union{ParamsVector{Float64}, Nothing} - parameters::Vector{Symbol} + params::Vector{Symbol} colnames::Union{Vector{Symbol}, Nothing} # better call it "variables": it's a mixture of observed and latent (and it gets confusing with get_colnames()) end @@ -55,7 +55,7 @@ end function RAMMatrices(; A::AbstractMatrix, S::AbstractMatrix, F::AbstractMatrix, M::Union{AbstractVector, Nothing} = nothing, - parameters::AbstractVector{Symbol}, + params::AbstractVector{Symbol}, colnames::Union{AbstractVector{Symbol}, Nothing} = nothing) ncols = size(A, 2) if !isnothing(colnames) @@ -69,14 +69,14 @@ function RAMMatrices(; A::AbstractMatrix, S::AbstractMatrix, if !isnothing(M) length(M) == ncols || throw(DimensionMismatch("M should have as many elements as colnames length ($ncols), $(length(M)) found")) end - A = ParamsMatrix{Float64}(A, parameters) - S = ParamsMatrix{Float64}(S, parameters) - M = !isnothing(M) ? ParamsVector{Float64}(M, parameters) : nothing + A = ParamsMatrix{Float64}(A, params) + S = ParamsMatrix{Float64}(S, params) + M = !isnothing(M) ? ParamsVector{Float64}(M, params) : nothing spF = sparse(F) if any(!isone, spF.nzval) throw(ArgumentError("F should contain only 0s and 1s")) end - return RAMMatrices(A, S, F, M, parameters, colnames) + return RAMMatrices(A, S, F, M, copy(params), colnames) end ############################################################################################ @@ -86,9 +86,7 @@ end function RAMMatrices(partable::ParameterTable; params::Union{AbstractVector{Symbol}, Nothing} = nothing) - if isnothing(params) - params = parameters(partable) - end + params = copy(isnothing(params) ? SEM.params(partable) : params) params_index = Dict(param => i for (i, param) in enumerate(params)) if length(params) != length(params_index) params_seen = Set{Symbol}() @@ -199,29 +197,47 @@ Base.convert(::Type{RAMMatrices}, partable::ParameterTable) = RAMMatrices(partab ### get parameter table from RAMMatrices ############################################################################################ -function ParameterTable(ram_matrices::RAMMatrices) +function ParameterTable(ram::RAMMatrices; + params::Union{AbstractVector{Symbol}, Nothing} = nothing, + observed_var_prefix::Symbol = :obs, + latent_var_prefix::Symbol = :var) + # defer parameter checks until we know which ones are used - colnames = ram_matrices.colnames + if !isnothing(ram.colnames) + latent_vars = SEM.latent_vars(ram) + observed_vars = SEM.observed_vars(ram) + colnames = ram.colnames + else + observed_vars = [Symbol("$(observed_var_prefix)_$i") for i in 1:nobserved_vars(ram)] + latent_vars = [Symbol("$(latent_var_prefix)_$i") for i in 1:nlatent_vars(ram)] + colnames = vcat(observed_vars, latent_vars) + end - partable = ParameterTable(observed_vars = colnames[ram_matrices.F.rowval], - latent_vars = colnames[setdiff(eachindex(colnames), - ram_matrices.F.rowval)]) + # construct an empty table + partable = ParameterTable(observed_vars = observed_vars, + latent_vars = latent_vars, + params = isnothing(params) ? ram.params : params) + # fill the table position_names = Dict{Int, Symbol}(1:length(colnames) .=> colnames) - append_rows!(partable, ram_matrices.A, :A, - ram_matrices.parameters, position_names) - append_rows!(partable, ram_matrices.S, :S, - ram_matrices.parameters, position_names, skip_symmetric=true) - if !isnothing(ram_matrices.M) - append_rows!(partable, ram_matrices.M, :M, - ram_matrices.parameters, position_names) + append_rows!(partable, ram.S, :S, + ram.params, position_names, skip_symmetric=true) + append_rows!(partable, ram.A, :A, + ram.params, position_names) + if !isnothing(ram.M) + append_rows!(partable, ram.M, :M, + ram.params, position_names) end + check_params(SEM.params(partable), partable.columns.param) + return partable end -Base.convert(::Type{<:ParameterTable}, ram_matrices::RAMMatrices) = ParameterTable(ram_matrices) +Base.convert(::Type{<:ParameterTable}, ram::RAMMatrices; + params::Union{AbstractVector{Symbol}, Nothing} = nothing) = + ParameterTable(ram; params = params) ############################################################################################ ### Pretty Printing @@ -236,23 +252,6 @@ end ### Additional Functions ############################################################################################ -# get the vector of all parameters in the table -# the position of the parameter is based on its first appearance in the table (and the ensemble) -function parameters(partable::Union{EnsembleParameterTable, ParameterTable}) - if partable isa ParameterTable - parameters = partable.columns.identifier - else - parameters = Vector{Symbol}() - for tbl in values(partable.tables) - append!(parameters, tbl.columns.identifier) - end - end - parameters = unique(parameters) - filter!(!=(:const), parameters) # exclude constants - - return parameters -end - function matrix_to_parameter_type(matrix::Symbol) if matrix == :A return :→ @@ -291,7 +290,7 @@ end function append_rows!(partable::ParameterTable, arr::ParamsArray, arr_name::Symbol, - parameters::AbstractVector, + params::AbstractVector, position_names; skip_symmetric::Bool = false) nparams(arr) == length(params) || @@ -300,7 +299,7 @@ function append_rows!(partable::ParameterTable, # add parameters visited_indices = Set{eltype(arr_ixs)}() - for (i, par) in enumerate(parameters) + for (i, par) in enumerate(params) for j in param_occurences_range(arr, i) arr_ix = arr_ixs[arr.linear_indices[j]] skip_symmetric && (arr_ix ∈ visited_indices) && continue diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 3f360f3b7..8c0599456 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -32,13 +32,15 @@ label(args...) = Label(args) function ParameterTable(graph::AbstractStenoGraph; observed_vars, latent_vars, - group::Integer = 1, param_prefix = :θ) + params::Union{AbstractVector{Symbol}, Nothing} = nothing, + group::Integer = 1, param_prefix::Symbol = :θ) graph = unique(graph) n = length(graph) partable = ParameterTable( latent_vars = latent_vars, - observed_vars = observed_vars) + observed_vars = observed_vars, + params = params) from = resize!(partable.columns.from, n) parameter_type = resize!(partable.columns.parameter_type, n) to = resize!(partable.columns.to, n) @@ -99,6 +101,19 @@ function ParameterTable(graph::AbstractStenoGraph; end end + if isnothing(params) + # collect the unique params in the order of appearance + @assert isempty(partable.params) + ids = Set{Symbol}() + for id in param + if (id != :const) && (id ∉ ids) + push!(ids, id) + push!(partable.params, id) + end + end + end + check_params(partable.params, param) + return partable end @@ -107,7 +122,8 @@ end ############################################################################################ function EnsembleParameterTable(graph::AbstractStenoGraph; - observed_vars, latent_vars, groups) + observed_vars, latent_vars, groups, + params::Union{AbstractVector{Symbol}, Nothing} = nothing) graph = unique(graph) @@ -115,8 +131,9 @@ function EnsembleParameterTable(graph::AbstractStenoGraph; graph; observed_vars = observed_vars, latent_vars = latent_vars, + params = params, group = i, param_prefix = Symbol(:g, group)) for (i, group) in enumerate(groups)) - return EnsembleParameterTable(partables) + return EnsembleParameterTable(partables, params = params) end \ No newline at end of file diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 61da17d8b..e20fed166 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -34,7 +34,7 @@ and for models with a meanstructure, the model implied means are computed as ``` ## Interfaces -- `identifier(::RAM) `-> Dict containing the parameter labels and their position +- `params(::RAM) `-> Dict containing the parameter labels and their position - `nparams(::RAM)` -> Number of parameters - `Σ(::RAM)` -> model implied covariance matrix @@ -65,7 +65,7 @@ Additional interfaces Only available in gradient! calls: - `I_A⁻¹(::RAM)` -> ``(I-A)^{-1}`` """ -mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, M1, M2, M3, M4, S1, S2, S3, D} <: SemImply{MS, ExactHessian} +mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, M1, M2, M3, M4, S1, S2, S3} <: SemImply{MS, ExactHessian} Σ::A1 A::A2 S::A3 @@ -83,8 +83,6 @@ mutable struct RAM{MS, A1, A2, A3, A4, A5, A6, V2, M1, M2, M3, M4, S1, S2, S3, D ∇A::S1 ∇S::S2 ∇M::S3 - - identifier::D end ############################################################################################ @@ -159,9 +157,7 @@ function RAM(; ∇A, ∇S, - ∇M, - - identifier(ram_matrices) + ∇M ) end @@ -169,11 +165,11 @@ end ### methods ############################################################################################ -function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, parameters) - materialize!(imply.A, imply.ram_matrices.A, parameters) - materialize!(imply.S, imply.ram_matrices.S, parameters) +function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingle, params) + materialize!(imply.A, imply.ram_matrices.A, params) + materialize!(imply.S, imply.ram_matrices.S, params) if !isnothing(imply.M) - materialize!(imply.M, imply.ram_matrices.M, parameters) + materialize!(imply.M, imply.ram_matrices.M, params) end @inbounds for (j, I_Aj, Aj) in zip(axes(imply.A, 2), eachcol(imply.I_A), eachcol(imply.A)) @@ -203,8 +199,7 @@ end ### Recommended methods ############################################################################################ -identifier(imply::RAM) = imply.identifier -nparams(imply::RAM) = nparams(imply.ram_matrices) +params(imply::RAM) = params(imply.ram_matrices) function update_observed(imply::RAM, observed::SemObserved; kwargs...) if n_man(observed) == size(imply.Σ, 1) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index b55a907c8..7e46f2088 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -29,7 +29,7 @@ Subtype of `SemImply` that implements the RAM notation with symbolic precomputat Subtype of `SemImply`. ## Interfaces -- `identifier(::RAMSymbolic) `-> Dict containing the parameter labels and their position +- `params(::RAMSymbolic) `-> vector of parameter names - `nparams(::RAMSymbolic)` -> number of parameters - `Σ(::RAMSymbolic)` -> model implied covariance matrix @@ -62,7 +62,7 @@ and for models with a meanstructure, the model implied means are computed as \mu = F(I-A)^{-1}M ``` """ -struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, D1} <: SemImplySymbolic{MS,ExactHessian} +struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5} <: SemImplySymbolic{MS,ExactHessian} Σ_function::F1 ∇Σ_function::F2 ∇²Σ_function::F3 @@ -77,7 +77,6 @@ struct RAMSymbolic{MS, F1, F2, F3, A1, A2, A3, S1, S2, S3, V2, F4, A4, F5, A5, D μ::A4 ∇μ_function::F5 ∇μ::A5 - identifier::D1 end ############################################################################################ @@ -185,7 +184,6 @@ function RAMSymbolic(; μ, ∇μ_function, ∇μ, - identifier(ram_matrices), ) end @@ -211,7 +209,7 @@ end ### Recommended methods ############################################################################################ -identifier(imply::RAMSymbolic) = imply.identifier +params(imply::RAMSymbolic) = params(imply.ram_matrices) nparams(imply::RAMSymbolic) = nparams(imply.ram_matrices) function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) diff --git a/src/imply/empty.jl b/src/imply/empty.jl index 3e77bef34..927e63bb4 100644 --- a/src/imply/empty.jl +++ b/src/imply/empty.jl @@ -19,15 +19,14 @@ model per group and an additional model with `ImplyEmpty` and `SemRidge` for the # Extended help ## Interfaces -- `identifier(::RAMSymbolic) `-> Dict containing the parameter labels and their position +- `params(::RAMSymbolic) `-> Dict containing the parameter labels and their position - `nparams(::RAMSymbolic)` -> Number of parameters ## Implementation Subtype of `SemImply`. """ -struct ImplyEmpty{V, V2} <: SemImply{NoMeanStructure,ExactHessian} - identifier::V2 - n_par::V +struct ImplyEmpty <: SemImply{NoMeanStructure,ExactHessian} + params::Vector{Symbol} end ############################################################################################ @@ -35,15 +34,10 @@ end ############################################################################################ function ImplyEmpty(; - specification, + specification::SemSpecification, kwargs...) - ram_matrices = RAMMatrices(specification) - identifier = identifier(ram_matrices) - - n_par = length(ram_matrices.parameters) - - return ImplyEmpty(identifier, n_par) + return ImplyEmpty(params(spec)) end ############################################################################################ @@ -56,7 +50,4 @@ update!(targets::EvaluationTargets, imply::ImplyEmpty, par, model) = nothing ### Recommended methods ############################################################################################ -identifier(imply::ImplyEmpty) = imply.identifier -nparams(imply::ImplyEmpty) = imply.nparams - update_observed(imply::ImplyEmpty, observed::SemObserved; kwargs...) = imply \ No newline at end of file diff --git a/src/loss/regularization/ridge.jl b/src/loss/regularization/ridge.jl index aa1c0ee3a..6c23b03ad 100644 --- a/src/loss/regularization/ridge.jl +++ b/src/loss/regularization/ridge.jl @@ -54,7 +54,8 @@ function SemRidge(; if isnothing(imply) throw(ArgumentError("When referring to parameters by label, `imply = ...` has to be specified")) else - which_ridge = get_identifier_indices(which_ridge, imply) + param_indices = Dict(param => i for (i, param) in enumerate(params(imply))) + which_ridge = [param_indices[param] for param in params(imply)] end end which_H = [CartesianIndex(x, x) for x in which_ridge] diff --git a/src/types.jl b/src/types.jl index cb60bf5fb..a44d4c2e5 100644 --- a/src/types.jl +++ b/src/types.jl @@ -180,14 +180,14 @@ Returns a SemEnsemble with fields - `sems::Tuple`: `AbstractSem`s. - `weights::Vector`: Weights for each model. - `optimizer::SemOptimizer`: Connects the model to the optimizer. See also [`SemOptimizer`](@ref). -- `identifier::Dict`: Stores parameter labels and their position. +- `params::Dict`: Stores parameter labels and their position. """ struct SemEnsemble{N, T <: Tuple, V <: AbstractVector, D, I} <: AbstractSemCollection n::N sems::T weights::V optimizer::D - identifier::I + params::I end function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing, kwargs...) @@ -200,11 +200,11 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing weights = [n_obs(model)/nobs_total for model in models] end - # check identifier equality - id = identifier(models[1]) + # check param equality + params1 = params(models[1]) for model in models - if id != identifier(model) - throw(ErrorException("The identifier of your models do not match. \n + if params1 != params(model) + throw(ErrorException("The parameters of your models do not match. \n Maybe you tried to specify models of an ensemble via ParameterTables. \n In that case, you may use RAMMatrices instead.")) end @@ -220,10 +220,12 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing models, weights, optimizer, - id + params1 ) end +params(ensemble::SemEnsemble) = ensemble.params + """ n_models(ensemble::SemEnsemble) -> Integer @@ -266,6 +268,8 @@ Returns the imply part of a model. """ imply(model::AbstractSemSingle) = model.imply +params(model::AbstractSemSingle) = params(imply(model)) + """ loss(model::AbstractSemSingle) -> SemLoss @@ -282,6 +286,9 @@ optimizer(model::AbstractSemSingle) = model.optimizer abstract type SemSpecification end +params(spec::SemSpecification) = spec.params +nparams(spec::SemSpecification) = length(params(spec)) + # observed + latent vars(spec::SemSpecification) = error("vars(spec::$(typeof(spec))) is not implemented") diff --git a/test/examples/helper.jl b/test/examples/helper.jl index 797490f35..efab6e7b5 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,6 +1,8 @@ using LinearAlgebra: norm function test_gradient(model, parameters; rtol = 1e-10, atol = 0) + @test nparams(model) == length(parameters) + true_grad = FiniteDiff.finite_difference_gradient(Base.Fix1(objective!, model), parameters) gradient = similar(parameters) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 7dbab094d..2ffd1c308 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -14,6 +14,8 @@ model_g2 = Sem( imply = RAM ) +@test SEM.params(model_g1.imply.ram_matrices) == SEM.params(model_g2.imply.ram_matrices) + model_ml_multigroup = SemEnsemble(model_g1, model_g2; optimizer = semoptimizer) diff --git a/test/examples/multigroup/multigroup.jl b/test/examples/multigroup/multigroup.jl index b551415c4..08d9c8462 100644 --- a/test/examples/multigroup/multigroup.jl +++ b/test/examples/multigroup/multigroup.jl @@ -50,14 +50,14 @@ specification_g1 = RAMMatrices(; A = A, S = S1, F = F, - parameters = x, + params = x, colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed]) specification_g2 = RAMMatrices(; A = A, S = S2, F = F, - parameters = x, + params = x, colnames = [:x1, :x2, :x3, :x4, :x5, :x6, :x7, :x8, :x9, :visual, :textual, :speed]) partable = EnsembleParameterTable( diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 8dfa86a46..5a5219522 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -9,6 +9,7 @@ model_ml = Sem( data = dat, optimizer = semoptimizer ) +@test SEM.params(model_ml.imply.ram_matrices) == SEM.params(spec) model_ml_cov = Sem( specification = spec, diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index c8c0cdb64..4c893ceae 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -1,5 +1,7 @@ using StructuralEquationModels, Test, FiniteDiff +SEM = StructuralEquationModels + include( joinpath(chop(dirname(pathof(StructuralEquationModels)), tail = 3), "test/examples/helper.jl") @@ -67,15 +69,15 @@ spec = RAMMatrices(; A = A, S = S, F = F, - parameters = x, + params = x, colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65] ) partable = ParameterTable(spec) -# w. meanstructure ------------------------------------------------------------------------- +@test SEM.params(spec) == SEM.params(partable) -x = Symbol.("x".*string.(1:38)) +# w. meanstructure ------------------------------------------------------------------------- M = [:x32; :x33; :x34; :x35; :x36; :x37; :x38; :x35; :x36; :x37; :x38; 0.0; 0.0; 0.0] @@ -84,11 +86,13 @@ spec_mean = RAMMatrices(; S = S, F = F, M = M, - parameters = x, + params = [SEM.params(spec); Symbol.("x", string.(32:38))], colnames = [:x1, :x2, :x3, :y1, :y2, :y3, :y4, :y5, :y6, :y7, :y8, :ind60, :dem60, :dem65]) partable_mean = ParameterTable(spec_mean) +@test SEM.params(partable_mean) == SEM.params(spec_mean) + start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3)] start_test_mean = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3); fill(0.1, 7)] @@ -114,6 +118,8 @@ end spec = ParameterTable(spec) spec_mean = ParameterTable(spec_mean) +@test SEM.params(spec) == SEM.params(partable) + partable = spec partable_mean = spec_mean diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 115a06721..7c6270583 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -32,7 +32,7 @@ A = [0 0 0 0 0 0 1.0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] -ram_matrices = RAMMatrices(;A = A, S = S, F = F, parameters = x, colnames = nothing) +ram_matrices = RAMMatrices(;A = A, S = S, F = F, params = x, colnames = nothing) true_val = [repeat([1], 8) 0.4 From 84bf448a325c7a3df80da2b8c6755907d0a0a264 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:52:54 -0700 Subject: [PATCH 094/174] identifier column -> param --- .../start_val/start_partable.jl | 2 +- src/frontend/fit/summary.jl | 38 +++++++++---------- src/frontend/specification/ParameterTable.jl | 8 ++-- src/frontend/specification/RAMMatrices.jl | 6 +-- src/frontend/specification/StenoGraphs.jl | 16 ++++---- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/additional_functions/start_val/start_partable.jl b/src/additional_functions/start_val/start_partable.jl index 038634f2b..5f90ae3c2 100644 --- a/src/additional_functions/start_val/start_partable.jl +++ b/src/additional_functions/start_val/start_partable.jl @@ -28,7 +28,7 @@ function start_parameter_table(ram::RAMMatrices; partable::ParameterTable, kwarg param_indices = Dict(param => i for (i, param) in enumerate(params(ram))) - for (i, param) in enumerate(partable.columns.identifier) + for (i, param) in enumerate(partable.columns.param) par_ind = get(param_indices, param, nothing) if !isnothing(par_ind) isfinite(partable.start[i]) && (start_val[i] = partable.start[i]) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index f3db9912b..dcfd819e1 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -51,49 +51,49 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co printstyled("Loadings: \n"; color = color) print("\n") - sorted_columns = [:to, :estimate, :identifier, :value_fixed, :start] + sorted_columns = [:to, :estimate, :param, :value_fixed, :start] loading_columns = sort_partially(sorted_columns, columns) header_cols = copy(loading_columns) replace!(header_cols, :parameter_type => :type) for var in partable.variables[:latent_vars] - indicator_indices = + indicator_indices = findall( - (partable.columns[:from] .== var) .& + (partable.columns[:from] .== var) .& (partable.columns[:parameter_type] .== :→) .& (partable.columns[:to] .∈ [partable.variables[:observed_vars]]) ) loading_array = reduce(hcat, check_round(partable.columns[c][indicator_indices]; digits = digits) for c in loading_columns) - + printstyled(var; color = secondary_color); print("\n") print("\n") pretty_table(loading_array; header = header_cols, tf = PrettyTables.tf_borderless, alignment = :l) print("\n") - + end printstyled("Directed Effects: \n"; color = color) - regression_indices = + regression_indices = findall( (partable.columns[:parameter_type] .== :→) .& ( ( - (partable.columns[:to] .∈ [partable.variables[:observed_vars]]) .& + (partable.columns[:to] .∈ [partable.variables[:observed_vars]]) .& (partable.columns[:from] .∈ [partable.variables[:observed_vars]]) ) .| ( - (partable.columns[:to] .∈ [partable.variables[:latent_vars]]) .& + (partable.columns[:to] .∈ [partable.variables[:latent_vars]]) .& (partable.columns[:from] .∈ [partable.variables[:observed_vars]]) ) .| ( - (partable.columns[:to] .∈ [partable.variables[:latent_vars]]) .& + (partable.columns[:to] .∈ [partable.variables[:latent_vars]]) .& (partable.columns[:from] .∈ [partable.variables[:latent_vars]]) ) ) ) - - sorted_columns = [:from, :parameter_type, :to, :estimate, :identifier, :value_fixed, :start] + + sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] regression_columns = sort_partially(sorted_columns, columns) regression_array = reduce(hcat, check_round(partable.columns[c][regression_indices]; digits = digits) for c in regression_columns) @@ -106,13 +106,13 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co printstyled("Variances: \n"; color = color) - variance_indices = + variance_indices = findall( (partable.columns[:parameter_type] .== :↔) .& (partable.columns[:to] .== partable.columns[:from]) ) - sorted_columns = [:from, :parameter_type, :to, :estimate, :identifier, :value_fixed, :start] + sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce(hcat, check_round(partable.columns[c][variance_indices]; digits = digits) for c in variance_columns) @@ -125,15 +125,15 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co printstyled("Covariances: \n"; color = color) - variance_indices = + variance_indices = findall( (partable.columns[:parameter_type] .== :↔) .& (partable.columns[:to] .!= partable.columns[:from]) ) - sorted_columns = [:from, :parameter_type, :to, :estimate, :identifier, :value_fixed, :start] + sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) - + variance_array = reduce(hcat, check_round(partable.columns[c][variance_indices]; digits = digits) for c in variance_columns) variance_columns[2] = Symbol("") replace!(variance_columns, :parameter_type => :type) @@ -142,7 +142,7 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co pretty_table(variance_array; header = variance_columns, tf = PrettyTables.tf_borderless, alignment = :l) print("\n") - mean_indices = + mean_indices = findall( (partable.columns[:parameter_type] .== :→) .& (partable.columns[:from] .== Symbol("1")) @@ -152,9 +152,9 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co printstyled("Means: \n"; color = color) - sorted_columns = [:from, :parameter_type, :to, :estimate, :identifier, :value_fixed, :start] + sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) - + variance_array = reduce(hcat, check_round(partable.columns[c][mean_indices]; digits = digits) for c in variance_columns) variance_columns[2] = Symbol("") replace!(variance_columns, :parameter_type => :type) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index ab177f7ed..c8685bcb4 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -109,7 +109,7 @@ function Base.show(io::IO, partable::ParameterTable) :start, :estimate, :se, - :identifier] + :param] shown_columns = filter!(col -> haskey(partable.columns, col) && length(partable.columns[col]) > 0, relevant_columns) @@ -138,7 +138,7 @@ ParameterTableRow = @NamedTuple begin to::Symbol free::Bool value_fixed::Any - identifier::Symbol + param::Symbol end Base.getindex(partable::ParameterTable, i::Integer) = @@ -147,7 +147,7 @@ Base.getindex(partable::ParameterTable, i::Integer) = to = partable.columns.to[i], free = partable.columns.free[i], value_fixed = partable.columns.value_fixed[i], - identifier = partable.columns.identifier[i], + param = partable.columns.param[i], ) Base.length(partable::ParameterTable) = length(first(partable.columns)) @@ -256,7 +256,7 @@ function update_partable!(partable::ParameterTable, fixed_values = partable.columns.value_fixed param_index = Dict(zip(params, eachindex(params))) resize!(coldata, length(partable)) - for (i, id) in enumerate(partable.columns.identifier) + for (i, id) in enumerate(partable.columns.param) coldata[i] = id != :const ? values[param_index[id]] : fixed_values[i] diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 95de6361c..57cf7849b 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -151,7 +151,7 @@ function RAMMatrices(partable::ParameterTable; error("Unsupported parameter type: $(row.parameter_type)") end else - par_ind = params_index[row.identifier] + par_ind = params_index[row.param] if (row.parameter_type == :→) && (row.from == Symbol("1")) push!(M_inds[par_ind], row_ind) elseif row.parameter_type == :→ @@ -285,7 +285,7 @@ function partable_row(val, index, matrix::Symbol, value_fixed = free ? 0.0 : val, start = 0.0, estimate = 0.0, - identifier = free ? val : :const) + param = free ? val : :const) end function append_rows!(partable::ParameterTable, @@ -331,7 +331,7 @@ end function Base.:(==)(mat1::RAMMatrices, mat2::RAMMatrices) res = ( (mat1.A == mat2.A) && (mat1.S == mat2.S) && (mat1.F == mat2.F) && (mat1.M == mat2.M) && - (mat1.parameters == mat2.parameters) && + (mat1.params == mat2.params) && (mat1.colnames == mat2.colnames) ) return res end diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 8c0599456..27c4f534a 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -47,7 +47,7 @@ function ParameterTable(graph::AbstractStenoGraph; free = fill!(resize!(partable.columns.free, n), true) value_fixed = fill!(resize!(partable.columns.value_fixed, n), NaN) start = fill!(resize!(partable.columns.start, n), NaN) - identifier = fill!(resize!(partable.columns.identifier, n), Symbol("")) + param = fill!(resize!(partable.columns.param, n), Symbol("")) # group = Vector{Symbol}(undef, n) # start_partable = zeros(Bool, n) @@ -80,24 +80,24 @@ function ParameterTable(graph::AbstractStenoGraph; if modval == :NaN throw(DomainError(NaN, "NaN is not allowed as a parameter label.")) end - identifier[i] = modval + param[i] = modval end end end end - # make identifiers for parameters that are not labeled + # assign identifiers for parameters that are not labeled current_id = 1 - for i in 1:length(identifier) - if identifier[i] == Symbol("") + for i in 1:length(param) + if param[i] == Symbol("") if free[i] - identifier[i] = Symbol(param_prefix, :_, current_id) + param[i] = Symbol(param_prefix, :_, current_id) current_id += 1 else - identifier[i] = :const + param[i] = :const end elseif !free[i] - @warn "You labeled a constant ($(identifier[i])=$(value_fixed[i])). Please check if the labels of your graph are correct." + @warn "You labeled a constant ($(param[i])=$(value_fixed[i])). Please check if the labels of your graph are correct." end end From ba1283fa04b147d4f550332a1a8e795877bc6167 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 16:27:52 -0700 Subject: [PATCH 095/174] check_params(): opt to append missing ones --- src/frontend/specification/ParameterTable.jl | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index c8685bcb4..759eaa458 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -38,12 +38,26 @@ function ParameterTable(; observed_vars::Union{AbstractVector{Symbol}, Nothing}= !isnothing(params) ? copy(params) : Vector{Symbol}()) end -function check_params(params::AbstractVector{Symbol}, partable_ids::AbstractVector{Symbol}) - all_refs = Set(id for id in partable_ids if id != :const) - undecl_params = setdiff(all_refs, params) +# checks if params vector contain all ids from param_refs +# and optionally appends missing ones +function check_params(params::AbstractVector{Symbol}, + param_refs::AbstractVector{Symbol}; + append::Bool = false) + param_refset = Set(id for id in param_refs if id != :const) + undecl_params = setdiff(param_refset, params) if !isempty(undecl_params) - throw(ArgumentError("The following $(length(undecl_params)) parameters present in the table, but are not declared: " * + if append # append preserving the order + appended_params = Set{Symbol}() + for param in param_refs + if (param ∈ undecl_params) && (param ∉ appended_params) + push!(params, param) + push!(appended_params, param) + end + end + else + throw(ArgumentError("$(length(undecl_params)) parameters present in the table, but are not declared: " * join(sort!(collect(undecl_params))))) + end end end From 9c416ac4203df4c5d16891e18992643eba11e23f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 16:28:31 -0700 Subject: [PATCH 096/174] EnsParTable ctor: enforce same params in tables --- .../specification/EnsembleParameterTable.jl | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index ef8b932ab..fa6a9ca64 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -25,12 +25,20 @@ function EnsembleParameterTable(spec_ensemble::AbstractDict{K, V}; group => convert(ParameterTable, spec; params = params) for (group, spec) in pairs(spec_ensemble)) - # collect all SEM parameters in ensemble if not specified - params = isnothing(params) ? - unique(mapreduce(SEM.params, vcat, - values(partables), init=Vector{Symbol}())) : - copy(params) - + if isnothing(params) + # collect all SEM parameters in ensemble if not specified + # and apply the set to all partables + params = unique(mapreduce(SEM.params, vcat, + values(partables), init=Vector{Symbol}())) + for partable in values(partables) + if partable.params != params + copyto!(resize!(partable.params, length(params)), params) + #throw(ArgumentError("The parameter sets of the SEM specifications in the ensemble do not match.")) + end + end + else + params = copy(params) + end return EnsembleParameterTable{K}(partables, params) end From 6f37d7f542ff3db81de2b2b69246e26c8d87936a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 16:29:31 -0700 Subject: [PATCH 097/174] ParTable(graph): use check_params() to update params --- src/frontend/specification/StenoGraphs.jl | 29 ++++++++--------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 27c4f534a..5bdf9ce93 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -47,7 +47,7 @@ function ParameterTable(graph::AbstractStenoGraph; free = fill!(resize!(partable.columns.free, n), true) value_fixed = fill!(resize!(partable.columns.value_fixed, n), NaN) start = fill!(resize!(partable.columns.start, n), NaN) - param = fill!(resize!(partable.columns.param, n), Symbol("")) + param_refs = fill!(resize!(partable.columns.param, n), Symbol("")) # params in the graph # group = Vector{Symbol}(undef, n) # start_partable = zeros(Bool, n) @@ -80,7 +80,7 @@ function ParameterTable(graph::AbstractStenoGraph; if modval == :NaN throw(DomainError(NaN, "NaN is not allowed as a parameter label.")) end - param[i] = modval + param_refs[i] = modval end end end @@ -88,31 +88,21 @@ function ParameterTable(graph::AbstractStenoGraph; # assign identifiers for parameters that are not labeled current_id = 1 - for i in 1:length(param) - if param[i] == Symbol("") + for i in eachindex(param_refs) + if param_refs[i] == Symbol("") if free[i] - param[i] = Symbol(param_prefix, :_, current_id) + param_refs[i] = Symbol(param_prefix, :_, current_id) current_id += 1 else - param[i] = :const + param_refs[i] = :const end elseif !free[i] - @warn "You labeled a constant ($(param[i])=$(value_fixed[i])). Please check if the labels of your graph are correct." + @warn "You labeled a constant ($(param_refs[i])=$(value_fixed[i])). Please check if the labels of your graph are correct." end end - if isnothing(params) - # collect the unique params in the order of appearance - @assert isempty(partable.params) - ids = Set{Symbol}() - for id in param - if (id != :const) && (id ∉ ids) - push!(ids, id) - push!(partable.params, id) - end - end - end - check_params(partable.params, param) + # append params referenced in the table if params not explicitly provided + check_params(partable.params, param_refs, append=isnothing(params)) return partable end @@ -135,5 +125,6 @@ function EnsembleParameterTable(graph::AbstractStenoGraph; group = i, param_prefix = Symbol(:g, group)) for (i, group) in enumerate(groups)) + return EnsembleParameterTable(partables, params = params) end \ No newline at end of file From bcd8551a28f8e45105c5232c840db6c12b4ae821 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:41:08 -0700 Subject: [PATCH 098/174] WIP SemImplyState --- src/types.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/types.jl b/src/types.jl index a44d4c2e5..e2fc8b8c5 100644 --- a/src/types.jl +++ b/src/types.jl @@ -107,6 +107,18 @@ HessianEvaluation(::Type{<:SemImply{MS,HE}}) where {MS, HE <: MeanStructure} = H "Subtype of SemImply for all objects that can serve as the imply field of a SEM and use some form of symbolic precomputation." abstract type SemImplySymbolic{MS,HE} <: SemImply{MS,HE} end +""" +State of `SemImply` that corresponds to the specific SEM parameter values. + +Contains the necessary vectors and matrices for calculating the SEM +objective, gradient and hessian (whichever is requested). +""" +abstract type SemImplyState end + +imply(state::SemImplyState) = state.imply +MeanStructure(state::SemImplyState) = MeanStructure(imply(state)) +ApproximateHessian(state::SemImplyState) = ApproximateHessian(imply(state)) + """ Sem(;observed = SemObservedData, imply = RAM, loss = SemML, optimizer = SemOptimizerOptim, kwargs...) From 23abc82592ae32684ca637f43dc57076c04404de Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 00:41:42 -0700 Subject: [PATCH 099/174] SemCov: stricter type checks --- src/observed/covariance.jl | 29 +++++++++------------------ test/unit_tests/data_input_formats.jl | 8 ++++---- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 332b8fbf0..23650e3f6 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -48,41 +48,30 @@ end function SemObservedCovariance(; specification::Union{SemSpecification, Nothing} = nothing, - obs_cov, + obs_cov::AbstractMatrix, - obs_colnames = nothing, - spec_colnames = nothing, + obs_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, + spec_colnames::Union{AbstractVector{Symbol}, Nothing} = nothing, - obs_mean = nothing, - meanstructure = false, + obs_mean::Union{AbstractVector, Nothing} = nothing, + meanstructure::Bool = false, - n_obs = nothing, + n_obs::Union{Number, Nothing} = nothing, kwargs...) - - if !meanstructure & !isnothing(obs_mean) - + if !meanstructure && !isnothing(obs_mean) throw(ArgumentError("observed means were passed, but `meanstructure = false`")) - - elseif meanstructure & isnothing(obs_mean) - + elseif meanstructure && isnothing(obs_mean) throw(ArgumentError("`meanstructure = true`, but no observed means were passed")) - end if isnothing(spec_colnames) && !isnothing(specification) spec_colnames = observed_vars(specification) end - if !isnothing(spec_colnames) & isnothing(obs_colnames) - + if !isnothing(spec_colnames) && isnothing(obs_colnames) throw(ArgumentError("no `obs_colnames` were specified")) - - elseif !isnothing(spec_colnames) & !(eltype(obs_colnames) <: Symbol) - - throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) - end if !isnothing(spec_colnames) diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index a83bb2527..226132a42 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -203,13 +203,13 @@ end SemObservedCovariance(specification = nothing, obs_cov = dat_cov, obs_mean = dat_mean) end -@test_throws UndefKeywordError(:specification) SemObservedCovariance(obs_cov = dat_cov) +#@test_throws UndefKeywordError(:specification) SemObservedCovariance(obs_cov = dat_cov) @test_throws ArgumentError("no `obs_colnames` were specified") begin SemObservedCovariance(specification = spec, obs_cov = dat_cov) end -@test_throws ArgumentError("please specify `obs_colnames` as a vector of Symbols") begin +@test_throws TypeError begin SemObservedCovariance(specification = spec, obs_cov = dat_cov, obs_colnames = names(dat)) end @@ -264,9 +264,9 @@ end SemObservedCovariance(specification = spec, obs_cov = dat_cov, meanstructure = true) end -@test_throws UndefKeywordError SemObservedCovariance(data = dat_matrix, meanstructure = true) +#@test_throws UndefKeywordError SemObservedCovariance(data = dat_matrix, meanstructure = true) -@test_throws UndefKeywordError SemObservedCovariance(obs_cov = dat_cov, meanstructure = true) +#@test_throws UndefKeywordError SemObservedCovariance(obs_cov = dat_cov, meanstructure = true) @test_throws ArgumentError("`meanstructure = true`, but no observed means were passed") begin SemObservedCovariance( From ee171f574c6854dd155a953c34468715e1059448 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 23:49:07 -0700 Subject: [PATCH 100/174] FIML: simplify index generation --- src/loss/ML/FIML.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 0c9e4071b..1b4eebd3a 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -60,9 +60,11 @@ function SemFIML(;observed, specification, kwargs...) nman = Int64(n_man(observed)) imp_inv = zeros(nman, nman) mult = similar.(inverses) - - ∇ind = vec(CartesianIndices(Array{Float64}(undef, nman, nman))) - ∇ind = [findall(x -> !(x[1] ∈ ind || x[2] ∈ ind), ∇ind) for ind in patterns_not(observed)] + + # linear indicies of co-observed variable pairs for each pattern + Σ_linind = LinearIndices((nman, nman)) + ∇ind = [[Σ_linind[CartesianIndex(x, y)] for x in ind, y in ind] + for ind in patterns_not(observed)] commutation_indices = get_commutation_lookup(nvars(specification)^2) From 05ddefd9a585bd8db50c9243f72315ed6eb2deb7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 17 Mar 2024 23:55:41 -0700 Subject: [PATCH 101/174] get_data(SemObserved): default implementation --- src/observed/data.jl | 1 - src/observed/missing.jl | 1 - src/types.jl | 2 ++ 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/observed/data.jl b/src/observed/data.jl index 565498497..678f33512 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -123,7 +123,6 @@ n_man(observed::SemObservedData) = observed.n_man ### additional methods ############################################################################################ -get_data(observed::SemObservedData) = observed.data obs_cov(observed::SemObservedData) = observed.obs_cov obs_mean(observed::SemObservedData) = observed.obs_mean data_rowwise(observed::SemObservedData) = observed.data_rowwise diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 0a7d09da6..05568eaf7 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -201,7 +201,6 @@ n_man(observed::SemObservedMissing) = observed.n_man ### Additional methods ############################################################################################ -get_data(observed::SemObservedMissing) = observed.data patterns(observed::SemObservedMissing) = observed.patterns patterns_not(observed::SemObservedMissing) = observed.patterns_not rows(observed::SemObservedMissing) = observed.rows diff --git a/src/types.jl b/src/types.jl index e2fc8b8c5..f7d18f267 100644 --- a/src/types.jl +++ b/src/types.jl @@ -93,6 +93,8 @@ If you have a special kind of data, e.g. ordinal data, you should implement a su """ abstract type SemObserved end +get_data(observed::SemObserved) = observed.data + """ Supertype of all objects that can serve as the imply field of a SEM. Computed model-implied values that should be compared with the observed data to find parameter estimates, From 9784e14388bed0961673e5747bed28b4ea77586b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 10 Apr 2024 15:38:18 -0700 Subject: [PATCH 102/174] SemObsMissing: refactor * use SemObsMissingPattern struct to simplify code * replace O(Nvars^2) common pattern detection with Dict{} * don't store row-wise, store sub-matrices of non-missing data instead * use StatsBase.mean_and_cov() --- src/frontend/fit/fitmeasures/minus2ll.jl | 5 +- src/loss/ML/FIML.jl | 94 ++++++-------- src/observed/EM.jl | 33 +++-- src/observed/missing.jl | 149 ++++++++++------------- 4 files changed, 126 insertions(+), 155 deletions(-) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 4c2416912..9488d1956 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -32,9 +32,8 @@ minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml: # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n function minus2ll(minimum::Number, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemFIML) - F = minimum - F *= n_obs(observed) - F += sum(log(2π)*observed.pattern_n_obs.*observed.pattern_nvar_obs) + F = minimum * n_obs(observed) + F += log(2π)*sum(pat -> n_obs(pat)*nobserved_vars(pat), observed.patterns) return F end diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 1b4eebd3a..070644d62 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -46,25 +46,26 @@ end ### Constructors ############################################################################################ -function SemFIML(;observed, specification, kwargs...) +function SemFIML(; observed::SemObservedMissing, specification, kwargs...) - inverses = broadcast(x -> zeros(x, x), Int64.(pattern_nvar_obs(observed))) + inverses = [zeros(nobserved_vars(pat), nobserved_vars(pat)) + for pat in observed.patterns] choleskys = Array{Cholesky{Float64,Array{Float64,2}},1}(undef, length(inverses)) - n_patterns = size(rows(observed), 1) + n_patterns = length(observed.patterns) logdets = zeros(n_patterns) - imp_mean = zeros.(Int64.(pattern_nvar_obs(observed))) - meandiff = zeros.(Int64.(pattern_nvar_obs(observed))) + imp_mean = [zeros(nobserved_vars(pat)) for pat in observed.patterns] + meandiff = [zeros(nobserved_vars(pat)) for pat in observed.patterns] - nman = Int64(n_man(observed)) + nman = n_man(observed) imp_inv = zeros(nman, nman) mult = similar.(inverses) # linear indicies of co-observed variable pairs for each pattern Σ_linind = LinearIndices((nman, nman)) - ∇ind = [[Σ_linind[CartesianIndex(x, y)] for x in ind, y in ind] - for ind in patterns_not(observed)] + ∇ind = [[Σ_linind[CartesianIndex(x, y)] for x in findall(pat.obs_mask), y in findall(pat.obs_mask)] + for pat in observed.patterns] commutation_indices = get_commutation_lookup(nvars(specification)^2) @@ -100,9 +101,8 @@ function evaluate!(objective, gradient, hessian, prepare_SemFIML!(semfiml, model) scale = inv(n_obs(observed(model))) - obs_rows = rows(observed(model)) - isnothing(objective) || (objective = scale*F_FIML(obs_rows, semfiml, model, parameters)) - isnothing(gradient) || (∇F_FIML!(gradient, obs_rows, semfiml, model); gradient .*= scale) + isnothing(objective) || (objective = scale*F_FIML(observed(model), semfiml, model, parameters)) + isnothing(gradient) || (∇F_FIML!(gradient, observed(model), semfiml, model); gradient .*= scale) return objective end @@ -111,7 +111,7 @@ end ### Recommended methods ############################################################################################ -update_observed(lossfun::SemFIML, observed::SemObserved; kwargs...) = +update_observed(lossfun::SemFIML, observed::SemObserved; kwargs...) = SemFIML(;observed = observed, kwargs...) ############################################################################################ @@ -126,16 +126,16 @@ function F_one_pattern(meandiff, inverse, obs_cov, logdet, N) return F*N end -function ∇F_one_pattern(μ_diff, Σ⁻¹, S, pattern, ∇ind, N, Jμ, JΣ, model) +function ∇F_one_pattern(μ_diff, Σ⁻¹, S, obs_mask, ∇ind, N, Jμ, JΣ, model) diff⨉inv = μ_diff'*Σ⁻¹ if N > one(N) JΣ[∇ind] .+= N*vec(Σ⁻¹*(I - S*Σ⁻¹ - μ_diff*diff⨉inv)) - @. Jμ[pattern] += (N*2*diff⨉inv)' + @. Jμ[obs_mask] += (N*2*diff⨉inv)' else JΣ[∇ind] .+= vec(Σ⁻¹*(I - μ_diff*diff⨉inv)) - @. Jμ[pattern] += (2*diff⨉inv)' + @. Jμ[obs_mask] += (2*diff⨉inv)' end end @@ -161,31 +161,31 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) G .-= ∇μ' * Jμ end -function F_FIML(rows, semfiml, model, parameters) +function F_FIML(observed::SemObservedMissing, semfiml, model, parameters) F = zero(eltype(parameters)) - for i = 1:size(rows, 1) + for (i, pat) in enumerate(observed.patterns) F += F_one_pattern( - semfiml.meandiff[i], + semfiml.meandiff[i], semfiml.inverses[i], - obs_cov(observed(model))[i], - semfiml.logdets[i], - pattern_n_obs(observed(model))[i]) + pat.obs_cov, + semfiml.logdets[i], + n_obs(pat)) end return F end -function ∇F_FIML!(G, rows, semfiml, model) - Jμ = zeros(Int64(n_man(model))) - JΣ = zeros(Int64(n_man(model)^2)) - - for i = 1:size(rows, 1) +function ∇F_FIML!(G, observed::SemObservedMissing, semfiml, model) + Jμ = zeros(n_man(model)) + JΣ = zeros(n_man(model)^2) + + for (i, pat) in enumerate(observed.patterns) ∇F_one_pattern( - semfiml.meandiff[i], - semfiml.inverses[i], - obs_cov(observed(model))[i], - patterns(observed(model))[i], + semfiml.meandiff[i], + semfiml.inverses[i], + pat.obs_cov, + pat.obs_mask, semfiml.∇ind[i], - pattern_n_obs(observed(model))[i], + n_obs(pat), Jμ, JΣ, model) @@ -198,35 +198,21 @@ function prepare_SemFIML!(semfiml, model) batch_cholesky!(semfiml, model) #batch_sym_inv_update!(semfiml, model) batch_inv!(semfiml, model) - for i in 1:size(pattern_n_obs(observed(model)), 1) - semfiml.meandiff[i] .= obs_mean(observed(model))[i] - semfiml.imp_mean[i] + for (i, pat) in enumerate(observed(model).patterns) + semfiml.meandiff[i] .= pat.obs_mean .- semfiml.imp_mean[i] end end -function copy_per_pattern!(inverses, source_inverses, means, source_means, patterns) - @views for i = 1:size(patterns, 1) - inverses[i] .= - source_inverses[ - patterns[i], - patterns[i]] - end - - @views for i = 1:size(patterns, 1) - means[i] .= - source_means[patterns[i]] +function copy_per_pattern!(fiml::SemFIML, model::AbstractSem) + Σ = imply(model).Σ + μ = imply(model).μ + data = observed(model) + @inbounds @views for (i, pat) in enumerate(data.patterns) + fiml.inverses[i] .= Σ[pat.obs_mask, pat.obs_mask] + fiml.imp_means[i] .= μ[pat.obs_mask] end end -copy_per_pattern!( - semfiml, - model::M where {M <: AbstractSem}) = - copy_per_pattern!( - semfiml.inverses, - imply(model).Σ, - semfiml.imp_mean, - imply(model).μ, - patterns(observed(model))) - function batch_cholesky!(semfiml, model) for i = 1:size(semfiml.inverses, 1) semfiml.choleskys[i] = cholesky!(Symmetric(semfiml.inverses[i])) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 042426c4c..f4f6510cd 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -36,9 +36,9 @@ function em_mvn( 𝔼xxᵀ_pre = zeros(n_man, n_man) ### precompute for full cases - if length(observed.patterns[1]) == observed.n_man - for row ∈ observed.rows[1] - row = observed.data_rowwise[row] + fullpat = observed.patterns[1] + if nmissed_vars(fullpat) == 0 + for row in eachrow(fullpat.data) 𝔼x_pre += row; 𝔼xxᵀ_pre += row*row'; end @@ -98,21 +98,27 @@ function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xx Σ = em_model.Σ # Compute the expected sufficient statistics - for i in 2:length(observed.pattern_n_obs) + for pat in observed.patterns + (nmissed_vars(pat) == 0) && continue # skip full cases # observed and unobserved vars - u = observed.patterns_not[i] - o = observed.patterns[i] + u = pat.miss_mask + o = pat.obs_mask # precompute for pattern - V = Σ[u, u] - Σ[u, o] * (Σ[o, o]\Σ[o, u]) + Σoo = Σ[o, o] + Σuo = Σ[u, o] + μu = μ[u] + μo = μ[o] + + V = Σ[u, u] - Σuo * (Σoo \ Σ[o, u]) # loop trough data - for row in observed.rows[i] - m = μ[u] + Σ[u, o] * ( Σ[o, o] \ (observed.data_rowwise[row]-μ[o]) ) + for rowdata in eachrow(pat.data) + m = μu + Σuo * ( Σoo \ (rowdata-μo) ) 𝔼xᵢ[u] = m - 𝔼xᵢ[o] = observed.data_rowwise[row] + 𝔼xᵢ[o] = rowdata 𝔼xxᵀᵢ[u, u] = 𝔼xᵢ[u] * 𝔼xᵢ[u]' + V 𝔼xxᵀᵢ[o, o] = 𝔼xᵢ[o] * 𝔼xᵢ[o]' 𝔼xxᵀᵢ[o, u] = 𝔼xᵢ[o] * 𝔼xᵢ[u]' @@ -158,9 +164,10 @@ end # use μ and Σ of full cases function start_em_observed(observed::SemObservedMissing; kwargs...) - if (length(observed.patterns[1]) == observed.n_man) & (observed.pattern_n_obs[1] > 1) - μ = copy(observed.obs_mean[1]) - Σ = copy(Symmetric(observed.obs_cov[1])) + fullpat = observed.patterns[1] + if (nmissed_vars(fullpat) == 0) && (n_obs(fullpat) > 1) + μ = copy(fullpat.obs_mean) + Σ = copy(Symmetric(fullpat.obs_cov)) if !isposdef(Σ) Σ = Matrix(Diagonal(Σ)) end diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 05568eaf7..3cc7663c7 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -9,6 +9,50 @@ mutable struct EmMVNModel{A, b, B} fitted::B end +# data associated with the specific pattern of missed variable +struct SemObservedMissingPattern{T,S} + obs_mask::BitVector # observed vars mask + miss_mask::BitVector # missing vars mask + nobserved::Int + nmissed::Int + rows::Vector{Int} # rows in original data + data::Matrix{T} # non-missing submatrix of data + + obs_mean::Vector{S} # means of observed vars + obs_cov::Matrix{S} # covariance of observed vars +end + +function SemObservedMissingPattern( + obs_mask::BitVector, + rows::AbstractVector{<:Integer}, + data::AbstractMatrix +) + T = nonmissingtype(eltype(data)) + + pat_data = convert(Matrix{T}, view(data, rows, obs_mask)) + if size(pat_data, 1) > 1 + pat_mean, pat_cov = mean_and_cov(pat_data, 1, corrected=false) + @assert size(pat_cov) == (size(pat_data, 2), size(pat_data, 2)) + else + pat_mean = reshape(pat_data[1, :], 1, :) + pat_cov = fill(zero(T), 1, 1) + end + + miss_mask = .!obs_mask + + return SemObservedMissingPattern{T, eltype(pat_mean)}( + obs_mask, miss_mask, + sum(obs_mask), sum(miss_mask), + rows, pat_data, + dropdims(pat_mean, dims=1), pat_cov) +end + +n_man(pat::SemObservedMissingPattern) = length(pat.obs_mask) +nobserved_vars(pat::SemObservedMissingPattern) = pat.nobserved +nmissed_vars(pat::SemObservedMissingPattern) = pat.nmissed + +n_obs(pat::SemObservedMissingPattern) = length(pat.rows) + """ For observed data with missing values. @@ -33,7 +77,7 @@ For observed data with missing values. - `get_data(::SemObservedMissing)` -> observed data - `data_rowwise(::SemObservedMissing)` -> observed data as vector per observation, with missing values deleted -- `patterns(::SemObservedMissing)` -> indices of non-missing variables per missing patterns +- `patterns(::SemObservedMissing)` -> indices of non-missing variables per missing patterns - `patterns_not(::SemObservedMissing)` -> indices of missing variables per missing pattern - `rows(::SemObservedMissing)` -> row indices of observed data points that belong to each pattern - `pattern_n_obs(::SemObservedMissing)` -> number of data points per pattern @@ -53,31 +97,15 @@ use this if you are sure your observed data is in the right format. ## Additional keyword arguments: - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object """ -mutable struct SemObservedMissing{ - A <: AbstractArray, - D <: AbstractFloat, - O <: AbstractFloat, - P <: Vector, - P2 <: Vector, - R <: Vector, - PD <: AbstractArray, - PO <: AbstractArray, - PVO <: AbstractArray, - A2 <: AbstractArray, - A3 <: AbstractArray, +struct SemObservedMissing{ + A <: AbstractMatrix, + P <: SemObservedMissingPattern, S <: EmMVNModel } <: SemObserved data::A - n_man::D - n_obs::O - patterns::P # missing patterns - patterns_not::P2 - rows::R # coresponding rows in data_rowwise - data_rowwise::PD # list of data - pattern_n_obs::PO # observed rows per pattern - pattern_nvar_obs::PVO # number of non-missing variables per pattern - obs_mean::A2 - obs_cov::A3 + n_man::Int + n_obs::Int + patterns::Vector{P} em_model::S end @@ -129,65 +157,26 @@ function SemObservedMissing(; data = Matrix(data) end - # remove persons with only missings - keep = Vector{Int64}() - for i = 1:size(data, 1) - if any(.!ismissing.(data[i, :])) - push!(keep, i) - end - end - data = data[keep, :] - - - n_obs, n_man = size(data) - # compute and store the different missing patterns with their rowindices - missings = ismissing.(data) - patterns = [missings[i, :] for i = 1:size(missings, 1)] - - patterns_cart = findall.(!, patterns) - data_rowwise = [data[i, patterns_cart[i]] for i = 1:n_obs] - data_rowwise = convert.(Array{Float64}, data_rowwise) - - remember = Vector{BitArray{1}}() - rows = [Vector{Int64}(undef, 0) for i = 1:size(patterns, 1)] - for i = 1:size(patterns, 1) - unknown = true - for j = 1:size(remember, 1) - if patterns[i] == remember[j] - push!(rows[j], i) - unknown = false - end - end - if unknown - push!(remember, patterns[i]) - push!(rows[size(remember, 1)], i) + # detect all different missing patterns with their row indices + pattern_to_rows = Dict{BitVector, Vector{Int}}() + for (i, datarow) in zip(axes(data, 1), eachrow(data)) + pattern = BitVector(.!ismissing.(datarow)) + if sum(pattern) > 0 # skip all-missing rows + pattern_rows = get!(() -> Vector{Int}(), pattern_to_rows, pattern) + push!(pattern_rows, i) end end - rows = rows[1:length(remember)] - n_patterns = size(rows, 1) - - # sort by number of missings - sort_n_miss = sortperm(sum.(remember)) - remember = remember[sort_n_miss] - remember_cart = findall.(!, remember) - remember_cart_not = findall.(remember) - rows = rows[sort_n_miss] - - pattern_n_obs = size.(rows, 1) - pattern_nvar_obs = length.(remember_cart) - - cov_mean = [cov_and_mean(data_rowwise[rows]) for rows in rows] - obs_cov = [cov_mean[1] for cov_mean in cov_mean] - obs_mean = [cov_mean[2] for cov_mean in cov_mean] + # process each pattern and sort from most to least number of observed vars + patterns = [SemObservedMissingPattern(pat, rows, data) + for (pat, rows) in pairs(pattern_to_rows)] + sort!(patterns, by=nmissed_vars) + # allocate EM model (but don't fit) em_model = EmMVNModel(zeros(n_man, n_man), zeros(n_man), false) - return SemObservedMissing(data, Float64(n_man), Float64(n_obs), remember_cart, - remember_cart_not, - rows, data_rowwise, Float64.(pattern_n_obs), Float64.(pattern_nvar_obs), - obs_mean, obs_cov, em_model) + return SemObservedMissing(data, n_man, n_obs, patterns, em_model) end ############################################################################################ @@ -200,13 +189,3 @@ n_man(observed::SemObservedMissing) = observed.n_man ############################################################################################ ### Additional methods ############################################################################################ - -patterns(observed::SemObservedMissing) = observed.patterns -patterns_not(observed::SemObservedMissing) = observed.patterns_not -rows(observed::SemObservedMissing) = observed.rows -data_rowwise(observed::SemObservedMissing) = observed.data_rowwise -pattern_n_obs(observed::SemObservedMissing) = observed.pattern_n_obs -pattern_nvar_obs(observed::SemObservedMissing) = observed.pattern_nvar_obs -obs_mean(observed::SemObservedMissing) = observed.obs_mean -obs_cov(observed::SemObservedMissing) = observed.obs_cov -em_model(observed::SemObservedMissing) = observed.em_model \ No newline at end of file From fe2bfeb2d9ceddb8852b056be01babe3b910e379 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 00:31:00 -0700 Subject: [PATCH 103/174] remove cov_and_mean(): not used anymore StatsBase.mean_and_cov() is used instead --- src/additional_functions/helper.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 99bcfb3a8..d5c633b20 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -110,13 +110,6 @@ function sparse_outer_mul!(C, A, B::Vector, ind) #computes A*S*B -> C, where ind end end -function cov_and_mean(rows; corrected = false) - obs_mean, obs_cov = StatsBase.mean_and_cov( - reduce(hcat, rows), 1, - corrected = corrected) - return obs_cov, obs_mean -end - function duplication_matrix(nobs) nobs = Int(nobs) n1 = Int(nobs*(nobs+1)*0.5) From 0dfde3d151b19780505d324df8713126d97dd627 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 10:51:05 -0700 Subject: [PATCH 104/174] minus2ll(): cleanup method signatures --- src/frontend/fit/fitmeasures/minus2ll.jl | 47 ++++++++++-------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 9488d1956..34d9911b5 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -9,31 +9,32 @@ function minus2ll end # Single Models ############################################################################################ -# SemFit splices loss functions ------------------------------------------------------------ -minus2ll(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = - minus2ll( - sem_fit, - sem_fit.model.observed, - sem_fit.model.imply, - sem_fit.model.optimizer, - sem_fit.model.loss.functions... - ) - -minus2ll(sem_fit::SemFit, obs, imp, optimizer, args...) = minus2ll(sem_fit.minimum, obs, imp, optimizer, args...) +minus2ll(fit::SemFit) = minus2ll(fit, fit.model) + +function minus2ll(fit::SemFit, model::AbstractSemSingle) + minimum = objective!(model, fit.solution) + return minus2ll(minimum, model) +end + +minus2ll(minimum::Number, model::AbstractSemSingle) = + sum(loss -> minus2ll(minimum, model, loss), model.loss.functions) # SemML ------------------------------------------------------------------------------------ -minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = - n_obs(obs)*(minimum + log(2π)*n_man(obs)) +function minus2ll(minimum::Number, model::AbstractSemSingle, loss_ml::SemML) + obs = observed(model) + return n_obs(obs)*(minimum + log(2π)*n_man(obs)) +end # WLS -------------------------------------------------------------------------------------- -minus2ll(minimum::Number, obs, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = +minus2ll(minimum::Number, model::AbstractSemSingle, loss_ml::SemWLS) = missing # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n -function minus2ll(minimum::Number, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemFIML) - F = minimum * n_obs(observed) - F += log(2π)*sum(pat -> n_obs(pat)*nobserved_vars(pat), observed.patterns) +function minus2ll(minimum::Number, model::AbstractSemSingle, loss_ml::SemFIML) + obs = observed(model)::SemObservedMissing + F = minimum * n_obs(obs) + F += log(2π)*sum(pat -> n_obs(pat)*nobserved_vars(pat), obs.patterns) return F end @@ -99,14 +100,4 @@ end # Collection ############################################################################################ -minus2ll(minimum, model::AbstractSemSingle) = - minus2ll(minimum, model.observed, model.imply, model.optimizer, model.loss.functions...) - -function minus2ll(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) - m2ll = 0.0 - for sem in sem_fit.model.sems - minimum = objective!(sem, sem_fit.solution) - m2ll += minus2ll(minimum, sem) - end - return m2ll -end \ No newline at end of file +minus2ll(fit::SemFit, model::SemEnsemble) = sum(Base.Fix1(minus2ll, fit), model.sems) From 182501496b6fc05367dadb12e1baa81da0a1f0a0 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 17:17:26 -0700 Subject: [PATCH 105/174] fix chi2 --- src/frontend/fit/fitmeasures/chi2.jl | 101 +++++++++++---------------- 1 file changed, 40 insertions(+), 61 deletions(-) diff --git a/src/frontend/fit/fitmeasures/chi2.jl b/src/frontend/fit/fitmeasures/chi2.jl index 20f1dbf6e..b95b2833f 100644 --- a/src/frontend/fit/fitmeasures/chi2.jl +++ b/src/frontend/fit/fitmeasures/chi2.jl @@ -1,91 +1,70 @@ """ - χ²(sem_fit::SemFit) + χ²(fit::SemFit) Return the χ² value. """ -function χ² end +χ²(fit::SemFit) = χ²(fit, fit.model) ############################################################################################ # Single Models ############################################################################################ -# SemFit splices loss functions ------------------------------------------------------------ -χ²(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = - χ²( - sem_fit, - sem_fit.model.observed, - sem_fit.model.imply, - sem_fit.model.optimizer, - sem_fit.model.loss.functions... - ) +χ²(fit::SemFit, model::AbstractSemSingle) = + sum(loss -> χ²(loss, fit, model), model.loss.functions) # RAM + SemML -χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemML) = - (n_obs(sem_fit)-1)*(sem_fit.minimum - logdet(observed.obs_cov) - observed.n_man) +χ²(lossfun::SemML, fit::SemFit, model::AbstractSemSingle) = + (n_obs(fit) - 1) * + (fit.minimum - logdet(obs_cov(observed(model))) - n_man(observed(model))) # bollen, p. 115, only correct for GLS weight matrix -χ²(sem_fit::SemFit, observed, imp::Union{RAM, RAMSymbolic}, optimizer, loss_ml::SemWLS) = - (n_obs(sem_fit)-1)*sem_fit.minimum +χ²(lossfun::SemWLS, fit::SemFit, model::AbstractSemSingle) = + (n_obs(fit) - 1) * fit.minimum # FIML -function χ²(sem_fit::SemFit, observed::SemObservedMissing, imp, optimizer, loss_ml::SemFIML) - ll_H0 = minus2ll(sem_fit) - ll_H1 = minus2ll(observed) - chi2 = ll_H0 - ll_H1 - return chi2 +function χ²(lossfun::SemFIML, fit::SemFit, model::AbstractSemSingle) + ll_H0 = minus2ll(fit) + ll_H1 = minus2ll(observed(model)) + return ll_H0 - ll_H1 end ############################################################################################ # Collections ############################################################################################ -# SemFit splices loss functions ------------------------------------------------------------ -χ²(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) = - χ²( - sem_fit, - sem_fit.model, - sem_fit.model.sems[1].loss.functions[1] - ) +function χ²(fit::SemFit, models::SemEnsemble) + isempty(models.sems) && return 0.0 -function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemWLS} - check_ensemble_length(model) - check_lossfun_types(model, L) - return (sum(n_obs.(model.sems))-1)*sem_fit.minimum -end + lossfun = models.sems[1].loss.functions[1] + # check that all models use the same single loss function + L = typeof(lossfun) + for (i, sem) in enumerate(models.sems) + if length(sem.loss.functions) > 1 + @error "Model for group #$i has $(length(sem.loss.functions)) loss functions. Only the single one is supported" + end + cur_lossfun = sem.loss.functions[1] + if !isa(cur_lossfun, L) + @error "Loss function for group #$i model is $(typeof(cur_lossfun)), expected $L. Heterogeneous loss functions are not supported" + end + end -function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemML} - check_ensemble_length(model) - check_lossfun_types(model, L) - F_G = sem_fit.minimum - F_G -= sum([w*(logdet(m.observed.obs_cov) + m.observed.n_man) for (w, m) in zip(model.weights, model.sems)]) - return (sum(n_obs.(model.sems))-1)*F_G + return χ²(lossfun, fit, models) end -function χ²(sem_fit::SemFit, model::SemEnsemble, lossfun::L) where {L <: SemFIML} - check_ensemble_length(model) - check_lossfun_types(model, L) - - ll_H0 = minus2ll(sem_fit) - ll_H1 = sum(minus2ll.(observed.(models(model)))) - chi2 = ll_H0 - ll_H1 - - return chi2 +function χ²(lossfun::SemWLS, fit::SemFit, models::SemEnsemble) + return (sum(n_obs, models.sems) - 1) * fit.minimum end -function check_ensemble_length(model) - for sem in model.sems - if length(sem.loss.functions) > 1 - @error "A model for one of the groups contains multiple loss functions." +function χ²(lossfun::SemML, fit::SemFit, models::SemEnsemble) + G = sum(zip(models.weights, models.sems)) do (w, model) + data = observed(model) + w*(logdet(obs_cov(data)) + n_man(data)) end - end + return (sum(n_obs, models.sems) - 1) * (fit.minimum - G) end -function check_lossfun_types(model, type) - for sem in model.sems - for lossfun in sem.loss.functions - if !isa(lossfun, type) - @error "Your model(s) contain multiple lossfunctions with differing types." - end - end - end -end \ No newline at end of file +function χ²(lossfun::SemFIML, fit::SemFit, models::SemEnsemble) + ll_H0 = minus2ll(fit) + ll_H1 = sum(minus2ll∘observed, models.sems) + return ll_H0 - ll_H1 +end From ee0f27391bb5bf9b871e59092a47cf1ccac63716 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 18 Mar 2024 23:50:04 -0700 Subject: [PATCH 106/174] fix RMSEA --- src/frontend/fit/fitmeasures/RMSEA.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/frontend/fit/fitmeasures/RMSEA.jl b/src/frontend/fit/fitmeasures/RMSEA.jl index f291f7f04..d76108856 100644 --- a/src/frontend/fit/fitmeasures/RMSEA.jl +++ b/src/frontend/fit/fitmeasures/RMSEA.jl @@ -5,11 +5,13 @@ Return the RMSEA. """ function RMSEA end -RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: AbstractSemSingle, O}) = +RMSEA(sem_fit::SemFit) = RMSEA(sem_fit, sem_fit.model) + +RMSEA(sem_fit::SemFit, model::AbstractSemSingle) = RMSEA(df(sem_fit), χ²(sem_fit), n_obs(sem_fit)) -RMSEA(sem_fit::SemFit{Mi, So, St, Mo, O} where {Mi, So, St, Mo <: SemEnsemble, O}) = - sqrt(length(sem_fit.model.sems))*RMSEA(df(sem_fit), χ²(sem_fit), n_obs(sem_fit)) +RMSEA(sem_fit::SemFit, model::SemEnsemble) = + sqrt(length(model.sems))*RMSEA(df(sem_fit), χ²(sem_fit), n_obs(sem_fit)) function RMSEA(df, chi2, n_obs) rmsea = (chi2 - df) / (n_obs*df) From 848a00fc565f88c5b42e9104096ab2f511f1f2b1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 11:25:42 -0700 Subject: [PATCH 107/174] FIML: update --- src/additional_functions/helper.jl | 6 - src/frontend/fit/fitmeasures/minus2ll.jl | 71 +++----- src/loss/ML/FIML.jl | 221 ++++++++++------------- 3 files changed, 115 insertions(+), 183 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index d5c633b20..dacbd9e23 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -65,12 +65,6 @@ function remove_all_missing(data::AbstractMatrix) return data[keep, :], keep end -function batch_inv!(fun, model) - for i = 1:size(fun.inverses, 1) - fun.inverses[i] .= LinearAlgebra.inv!(fun.choleskys[i]) - end -end - #= function batch_sym_inv_update!(fun::Union{LossFunction, DiffFunction}, model) M_inv = inv(fun.choleskys[1]) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 34d9911b5..6caf5b962 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -12,26 +12,26 @@ function minus2ll end minus2ll(fit::SemFit) = minus2ll(fit, fit.model) function minus2ll(fit::SemFit, model::AbstractSemSingle) - minimum = objective!(model, fit.solution) + minimum = objective(model, fit.solution) return minus2ll(minimum, model) end minus2ll(minimum::Number, model::AbstractSemSingle) = - sum(loss -> minus2ll(minimum, model, loss), model.loss.functions) + sum(lossfun -> minus2ll(lossfun, minimum, model), model.loss.functions) # SemML ------------------------------------------------------------------------------------ -function minus2ll(minimum::Number, model::AbstractSemSingle, loss_ml::SemML) +function minus2ll(lossfun::SemML, minimum::Number, model::AbstractSemSingle) obs = observed(model) return n_obs(obs)*(minimum + log(2π)*n_man(obs)) end # WLS -------------------------------------------------------------------------------------- -minus2ll(minimum::Number, model::AbstractSemSingle, loss_ml::SemWLS) = +minus2ll(lossfun::SemWLS, minimum::Number, model::AbstractSemSingle) = missing # compute likelihood for missing data - H0 ------------------------------------------------- # -2ll = (∑ log(2π)*(nᵢ + mᵢ)) + F*n -function minus2ll(minimum::Number, model::AbstractSemSingle, loss_ml::SemFIML) +function minus2ll(lossfun::SemFIML, minimum::Number, model::AbstractSemSingle) obs = observed(model)::SemObservedMissing F = minimum * n_obs(obs) F += log(2π)*sum(pat -> n_obs(pat)*nobserved_vars(pat), obs.patterns) @@ -41,59 +41,30 @@ end # compute likelihood for missing data - H1 ------------------------------------------------- # -2ll = ∑ log(2π)*(nᵢ + mᵢ) + ln(Σᵢ) + (mᵢ - μᵢ)ᵀ Σᵢ⁻¹ (mᵢ - μᵢ)) + tr(SᵢΣᵢ) function minus2ll(observed::SemObservedMissing) - if observed.em_model.fitted - minus2ll( - observed.em_model.μ, - observed.em_model.Σ, - observed.n_obs, - observed.rows, - observed.patterns, - observed.obs_mean, - observed.obs_cov, - observed.pattern_n_obs, - observed.pattern_nvar_obs) - else - em_mvn(observed) - minus2ll( - observed.em_model.μ, - observed.em_model.Σ, - observed.n_obs, - observed.rows, - observed.patterns, - observed.obs_mean, - observed.obs_cov, - observed.pattern_n_obs, - observed.pattern_nvar_obs) - end -end + observed.em_model.fitted || em_mvn(observed) -function minus2ll(μ, Σ, N, rows, patterns, obs_mean, obs_cov, pattern_n_obs, pattern_nvar_obs) + μ = observed.em_model.μ + Σ = observed.em_model.Σ F = 0.0 - - for i in 1:length(rows) - - nᵢ = pattern_n_obs[i] - # missing pattern - pattern = patterns[i] - # observed data - Sᵢ = obs_cov[i] - + for pat in observed.patterns + nᵢ = n_obs(pat) # implied covariance/mean - Σᵢ = Σ[pattern, pattern] - ld = logdet(Σᵢ) - Σᵢ⁻¹ = inv(cholesky(Σᵢ)) - meandiffᵢ = obs_mean[i] - μ[pattern] + Σᵢ = Σ[pat.obs_mask, pat.obs_mask] - F += F_one_pattern(meandiffᵢ, Σᵢ⁻¹, Sᵢ, ld, nᵢ) - + ld = logdet(Σᵢ) + Σᵢ⁻¹ = LinearAlgebra.inv!(cholesky!(Σᵢ)) + μ_diffᵢ = pat.obs_mean - μ[pat.obs_mask] + + F_pat = ld + dot(μ_diffᵢ, Σᵢ⁻¹, μ_diffᵢ) + if n_obs(pat) > 1 + F_pat += dot(pat.obs_cov, Σᵢ⁻¹) + end + F += (F_pat + log(2π)*nobserved_vars(pat))*n_obs(pat) end - F += sum(log(2π)*pattern_n_obs.*pattern_nvar_obs) - #F *= N - + #F *= n_obs(observed) return F - end ############################################################################################ diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index 070644d62..a0ce598fd 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -1,6 +1,68 @@ ############################################################################################ ### Types ############################################################################################ + +# state of SemFIML for a specific missing pattern (`SemObservedMissingPattern` type) +struct SemFIMLPattern{T} + ∇ind::Vector{Int} # indices of co-observed variable pairs + Σ⁻¹::Matrix{T} # preallocated inverse of implied cov + logdet::Ref{T} # logdet of implied cov + μ_diff::Vector{T} # implied mean difference +end + +# allocate arrays for pattern FIML +function SemFIMLPattern(pat::SemObservedMissingPattern) + nobserved = nobserved_vars(pat) + nmissed = nmissed_vars(pat) + + # linear indicies of co-observed variable pairs for each pattern + Σ_linind = LinearIndices((n_man(pat), n_man(pat))) + ∇ind = vec([Σ_linind[CartesianIndex(x, y)] + for x in findall(pat.obs_mask), y in findall(pat.obs_mask)]) + + return SemFIMLPattern(∇ind, + zeros(nobserved, nobserved), + Ref(NaN), zeros(nobserved)) +end + +function prepare!(fiml::SemFIMLPattern, pat::SemObservedMissingPattern, implied::SemImply) + Σ = implied.Σ + μ = implied.μ + @inbounds @. @views begin + fiml.Σ⁻¹ = Σ[pat.obs_mask, pat.obs_mask] + fiml.μ_diff = pat.obs_mean - μ[pat.obs_mask] + end + Σ_chol = cholesky!(Symmetric(fiml.Σ⁻¹)) + fiml.logdet[] = logdet(Σ_chol) + LinearAlgebra.inv!(Σ_chol) # updates fiml.Σ⁻¹ + #batch_sym_inv_update!(fiml, model) + return fiml +end + +function objective(fiml::SemFIMLPattern{T}, pat::SemObservedMissingPattern) where T + F = fiml.logdet[] + dot(fiml.μ_diff, fiml.Σ⁻¹, fiml.μ_diff) + if n_obs(pat) > 1 + F += dot(pat.obs_cov, fiml.Σ⁻¹) + F *= n_obs(pat) + end + return F +end + +function gradient!(JΣ, Jμ, fiml::SemFIMLPattern, pat::SemObservedMissingPattern) + Σ⁻¹ = Symmetric(fiml.Σ⁻¹) + μ_diff⨉Σ⁻¹ = fiml.μ_diff' * Σ⁻¹ + if n_obs(pat) > 1 + JΣ_pat = Σ⁻¹ * (I - pat.obs_cov * Σ⁻¹ - fiml.μ_diff * μ_diff⨉Σ⁻¹) + JΣ_pat .*= n_obs(pat) + else + JΣ_pat = Σ⁻¹ * (I - fiml.μ_diff * μ_diff⨉Σ⁻¹) + end + Jμ_pat = ((2*n_obs(pat))*μ_diff⨉Σ⁻¹)' + vec(JΣ)[fiml.∇ind] .+= vec(JΣ_pat) + Jμ[pat.obs_mask] .+= Jμ_pat + return nothing +end + """ Full information maximum likelihood estimation. Can handle observed data with missings. @@ -24,20 +86,12 @@ Analytic gradients are available. ## Implementation Subtype of `SemLossFunction`. """ -mutable struct SemFIML{INV, C, L, O, M, IM, I, T, U, W} <: SemLossFunction{ExactHessian} - inverses::INV #preallocated inverses of imp_cov - choleskys::C #preallocated choleskys - logdets::L #logdets of implied covmats - - ∇ind::O +struct SemFIML{T, W} <: SemLossFunction{ExactHessian} + patterns::Vector{SemFIMLPattern{T}} - imp_mean::IM - meandiff::M - imp_inv::I + imp_inv::Matrix{T} # implied inverse - mult::T - - commutation_indices::U + commutation_indices::Dict{Int, Int} interaction::W end @@ -47,40 +101,9 @@ end ############################################################################################ function SemFIML(; observed::SemObservedMissing, specification, kwargs...) - - inverses = [zeros(nobserved_vars(pat), nobserved_vars(pat)) - for pat in observed.patterns] - choleskys = Array{Cholesky{Float64,Array{Float64,2}},1}(undef, length(inverses)) - - n_patterns = length(observed.patterns) - logdets = zeros(n_patterns) - - imp_mean = [zeros(nobserved_vars(pat)) for pat in observed.patterns] - meandiff = [zeros(nobserved_vars(pat)) for pat in observed.patterns] - - nman = n_man(observed) - imp_inv = zeros(nman, nman) - mult = similar.(inverses) - - # linear indicies of co-observed variable pairs for each pattern - Σ_linind = LinearIndices((nman, nman)) - ∇ind = [[Σ_linind[CartesianIndex(x, y)] for x in findall(pat.obs_mask), y in findall(pat.obs_mask)] - for pat in observed.patterns] - - commutation_indices = get_commutation_lookup(nvars(specification)^2) - - return SemFIML( - inverses, - choleskys, - logdets, - ∇ind, - imp_mean, - meandiff, - imp_inv, - mult, - commutation_indices, - nothing - ) + return SemFIML([SemFIMLPattern(pat) for pat in observed.patterns], + zeros(n_man(observed), n_man(observed)), + get_commutation_lookup(nvars(specification)^2), nothing) end ############################################################################################ @@ -88,21 +111,21 @@ end ############################################################################################ function evaluate!(objective, gradient, hessian, - semfiml::SemFIML, implied::SemImply, model::AbstractSemSingle, parameters) + fiml::SemFIML, implied::SemImply, model::AbstractSemSingle, params) isnothing(hessian) || error("Hessian not implemented for FIML") - if !check_fiml(semfiml, model) - isnothing(objective) || (objective = non_posdef_return(parameters)) + if !check(fiml, model) + isnothing(objective) || (objective = non_posdef_return(params)) isnothing(gradient) || fill!(gradient, 1) return objective end - prepare_SemFIML!(semfiml, model) + prepare!(fiml, model) scale = inv(n_obs(observed(model))) - isnothing(objective) || (objective = scale*F_FIML(observed(model), semfiml, model, parameters)) - isnothing(gradient) || (∇F_FIML!(gradient, observed(model), semfiml, model); gradient .*= scale) + isnothing(objective) || (objective = scale*F_FIML(eltype(params), fiml, observed(model), model)) + isnothing(gradient) || (∇F_FIML!(gradient, fiml, observed(model), model); gradient .*= scale) return objective end @@ -118,40 +141,18 @@ update_observed(lossfun::SemFIML, observed::SemObserved; kwargs...) = ### additional functions ############################################################################################ -function F_one_pattern(meandiff, inverse, obs_cov, logdet, N) - F = logdet + dot(meandiff, inverse, meandiff) - if N > one(N) - F += dot(obs_cov, inverse) - end - return F*N -end - -function ∇F_one_pattern(μ_diff, Σ⁻¹, S, obs_mask, ∇ind, N, Jμ, JΣ, model) - diff⨉inv = μ_diff'*Σ⁻¹ - - if N > one(N) - JΣ[∇ind] .+= N*vec(Σ⁻¹*(I - S*Σ⁻¹ - μ_diff*diff⨉inv)) - @. Jμ[obs_mask] += (N*2*diff⨉inv)' - - else - JΣ[∇ind] .+= vec(Σ⁻¹*(I - μ_diff*diff⨉inv)) - @. Jμ[obs_mask] += (2*diff⨉inv)' - end - -end - -function ∇F_fiml_outer!(G, JΣ, Jμ, imply::SemImplySymbolic, model, semfiml) +function ∇F_fiml_outer!(G, JΣ, Jμ, fiml::SemFIML, imply::SemImplySymbolic, model) mul!(G, imply.∇Σ', JΣ) # should be transposed G .-= imply.∇μ' * Jμ end -function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) +function ∇F_fiml_outer!(G, JΣ, Jμ, fiml::SemFIML, imply, model) Iₙ = sparse(1.0I, size(imply.A)...) P = kron(imply.F⨉I_A⁻¹, imply.F⨉I_A⁻¹) Q = kron(imply.S*imply.I_A⁻¹', Iₙ) #commutation_matrix_pre_square_add!(Q, Q) - Q2 = commutation_matrix_pre_square(Q, semfiml.commutation_indices) + Q2 = commutation_matrix_pre_square(Q, fiml.commutation_indices) ∇Σ = P*(imply.∇S + (Q+Q2)*imply.∇A) @@ -161,68 +162,34 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, imply, model, semfiml) G .-= ∇μ' * Jμ end -function F_FIML(observed::SemObservedMissing, semfiml, model, parameters) - F = zero(eltype(parameters)) - for (i, pat) in enumerate(observed.patterns) - F += F_one_pattern( - semfiml.meandiff[i], - semfiml.inverses[i], - pat.obs_cov, - semfiml.logdets[i], - n_obs(pat)) +function F_FIML(::Type{T}, fiml::SemFIML, observed::SemObservedMissing, model::AbstractSemSingle) where T + F = zero(T) + for (pat_fiml, pat) in zip(fiml.patterns, observed.patterns) + F += objective(pat_fiml, pat) end return F end -function ∇F_FIML!(G, observed::SemObservedMissing, semfiml, model) +function ∇F_FIML!(G, fiml::SemFIML, observed::SemObservedMissing, model::AbstractSemSingle) Jμ = zeros(n_man(model)) JΣ = zeros(n_man(model)^2) - for (i, pat) in enumerate(observed.patterns) - ∇F_one_pattern( - semfiml.meandiff[i], - semfiml.inverses[i], - pat.obs_cov, - pat.obs_mask, - semfiml.∇ind[i], - n_obs(pat), - Jμ, - JΣ, - model) - end - ∇F_fiml_outer!(G, JΣ, Jμ, imply(model), model, semfiml) -end - -function prepare_SemFIML!(semfiml, model) - copy_per_pattern!(semfiml, model) - batch_cholesky!(semfiml, model) - #batch_sym_inv_update!(semfiml, model) - batch_inv!(semfiml, model) - for (i, pat) in enumerate(observed(model).patterns) - semfiml.meandiff[i] .= pat.obs_mean .- semfiml.imp_mean[i] - end -end - -function copy_per_pattern!(fiml::SemFIML, model::AbstractSem) - Σ = imply(model).Σ - μ = imply(model).μ - data = observed(model) - @inbounds @views for (i, pat) in enumerate(data.patterns) - fiml.inverses[i] .= Σ[pat.obs_mask, pat.obs_mask] - fiml.imp_means[i] .= μ[pat.obs_mask] + for (pat_fiml, pat) in zip(fiml.patterns, observed.patterns) + gradient!(JΣ, Jμ, pat_fiml, pat) end + ∇F_fiml_outer!(G, JΣ, Jμ, fiml, imply(model), model) end -function batch_cholesky!(semfiml, model) - for i = 1:size(semfiml.inverses, 1) - semfiml.choleskys[i] = cholesky!(Symmetric(semfiml.inverses[i])) - semfiml.logdets[i] = logdet(semfiml.choleskys[i]) +function prepare!(fiml::SemFIML, model::AbstractSemSingle) + data = observed(model)::SemObservedMissing + @inbounds for (pat_fiml, pat) in zip(fiml.patterns, data.patterns) + prepare!(pat_fiml, pat, imply(model)) end - return true + #batch_sym_inv_update!(fiml, model) end -function check_fiml(semfiml, model) - copyto!(semfiml.imp_inv, imply(model).Σ) - a = cholesky!(Symmetric(semfiml.imp_inv); check = false) +function check(fiml::SemFIML, model::AbstractSemSingle) + copyto!(fiml.imp_inv, imply(model).Σ) + a = cholesky!(Symmetric(fiml.imp_inv); check = false) return isposdef(a) end From 1d0e8b510e2a26c2869a92fed72bc507f0a98623 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 12:25:31 -0700 Subject: [PATCH 108/174] FIML: use 5-arg mul! to avoid extra allocation --- src/loss/ML/FIML.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index a0ce598fd..cd9466e69 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -143,7 +143,7 @@ update_observed(lossfun::SemFIML, observed::SemObserved; kwargs...) = function ∇F_fiml_outer!(G, JΣ, Jμ, fiml::SemFIML, imply::SemImplySymbolic, model) mul!(G, imply.∇Σ', JΣ) # should be transposed - G .-= imply.∇μ' * Jμ + mul!(G, imply.∇μ', Jμ, -1, 1) end function ∇F_fiml_outer!(G, JΣ, Jμ, fiml::SemFIML, imply, model) @@ -159,7 +159,7 @@ function ∇F_fiml_outer!(G, JΣ, Jμ, fiml::SemFIML, imply, model) ∇μ = imply.F⨉I_A⁻¹*imply.∇M + kron((imply.I_A⁻¹*imply.M)', imply.F⨉I_A⁻¹)*imply.∇A mul!(G, ∇Σ', JΣ) # actually transposed - G .-= ∇μ' * Jμ + mul!(G, ∇μ', Jμ, -1, 1) end function F_FIML(::Type{T}, fiml::SemFIML, observed::SemObservedMissing, model::AbstractSemSingle) where T From 87a93f8238ec0168e9cdcbd2341c9133b2fff45c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 12:24:56 -0700 Subject: [PATCH 109/174] WLS: use 5-arg mul!() to reduce allocations --- src/loss/WLS/WLS.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index bb9647209..60f63c366 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -107,7 +107,7 @@ function evaluate!(objective, gradient, hessian, end gradient .*= -2 end - isnothing(hessian) || (mul!(hessian, ∇σ'*V, ∇σ); hessian .*= 2) + isnothing(hessian) || (mul!(hessian, ∇σ'*V, ∇σ, 2, 0)) if !isnothing(hessian) && (HessianEvaluation(semwls) === ExactHessian) ∇²Σ_function! = implied.∇²Σ_function ∇²Σ = implied.∇²Σ @@ -124,7 +124,7 @@ function evaluate!(objective, gradient, hessian, objective += dot(μ₋, V_μ, μ₋) end if !isnothing(gradient) - gradient .-= 2*(μ₋'*V_μ*implied.∇μ)' + mul!(gradient, (V_μ*implied.∇μ)', μ₋, -2, 1) end end From 9a13e7c9f71525be2525943f0c667466e7c6b088 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 16:10:26 -0700 Subject: [PATCH 110/174] ML: use 5-arg mul!() to reduce allocations --- src/loss/ML/ML.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 7681493ca..6ae3a0771 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -100,19 +100,19 @@ function evaluate!( ∇Σ = implied.∇Σ ∇μ = implied.∇μ μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ - gradient .= (vec(Σ⁻¹*(I - Σₒ*Σ⁻¹ - μ₋*μ₋ᵀΣ⁻¹))' * ∇Σ)' - gradient .-= (2*μ₋ᵀΣ⁻¹*∇μ)' + mul!(gradient, ∇Σ', vec(Σ⁻¹*(I - Σₒ*Σ⁻¹ - μ₋*μ₋ᵀΣ⁻¹))) + mul!(gradient, ∇μ', μ₋ᵀΣ⁻¹', -2, 1) end elseif !isnothing(gradient) || !isnothing(hessian) ∇Σ = implied.∇Σ Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' if !isnothing(gradient) - gradient .= (J*∇Σ)' + mul!(gradient, ∇Σ', J') end if !isnothing(hessian) if HessianEvaluation(semml) === ApproximateHessian - mul!(hessian, 2*∇Σ'*kron(Σ⁻¹, Σ⁻¹), ∇Σ) + mul!(hessian, ∇Σ'*kron(Σ⁻¹, Σ⁻¹), ∇Σ, 2, 0) else ∇²Σ_function! = implied.∇²Σ_function ∇²Σ = implied.∇²Σ @@ -179,7 +179,8 @@ function evaluate!( ∇S = implied.∇S C = F⨉I_A⁻¹'*(I-Σ⁻¹Σₒ)*Σ⁻¹*F⨉I_A⁻¹ - gradᵀ = 2vec(C*S*I_A⁻¹')'∇A + vec(C)'∇S + mul!(gradient, ∇A', vec(C*S*I_A⁻¹'), 2, 0) + mul!(gradient, ∇S', vec(C), 1, 1) if MeanStructure(implied) === HasMeanStructure μ = implied.μ @@ -189,9 +190,10 @@ function evaluate!( μ₋ = μₒ - μ μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ - gradᵀ .+= -2k*∇M - 2vec(k'*(M'+k*S)*I_A⁻¹')'∇A - vec(k'k)'∇S + mul!(gradient, ∇M', k', -2, 1) + mul!(gradient, ∇A', vec(k'*(I_A⁻¹*(M + S*k'))'), -2, 1) + mul!(gradient, ∇S', vec(k'k), -1, 1) end - copyto!(gradient, gradᵀ') end return objective From c4c4c8ce54f5e95a165a312f6b2a7d521219d7ff Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 18:32:18 -0700 Subject: [PATCH 111/174] declare cov matrices symmetric --- src/frontend/fit/fitmeasures/minus2ll.jl | 2 +- src/loss/ML/ML.jl | 4 ++-- src/observed/covariance.jl | 2 +- src/observed/data.jl | 4 ++-- src/observed/missing.jl | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 6caf5b962..6d6b61997 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -50,7 +50,7 @@ function minus2ll(observed::SemObservedMissing) for pat in observed.patterns nᵢ = n_obs(pat) # implied covariance/mean - Σᵢ = Σ[pat.obs_mask, pat.obs_mask] + Σᵢ = Symmetric(Σ[pat.obs_mask, pat.obs_mask]) ld = logdet(Σᵢ) Σᵢ⁻¹ = LinearAlgebra.inv!(cholesky!(Σᵢ)) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 6ae3a0771..1af993246 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -48,7 +48,7 @@ function SemML(; observed::SemObserved, meandiff = isnothing(obsmean) ? nothing : copy(obsmean) return SemML{approximate_hessian ? ApproximateHessian : ExactHessian}( - similar(obscov), similar(obscov), + similar(parent(obscov)), similar(parent(obscov)), meandiff) end @@ -66,7 +66,7 @@ function evaluate!( model::AbstractSemSingle, par) - if !isnothing(hessian) + if !isnothing(hessian) (MeanStructure(implied) === HasMeanStructure) && throw(DomainError(H, "hessian of ML + meanstructure is not available")) end diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 23650e3f6..6b99d6d4c 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -81,7 +81,7 @@ function SemObservedCovariance(; n_man = Float64(size(obs_cov, 1)) - return SemObservedCovariance(obs_cov, obs_mean, n_man, n_obs) + return SemObservedCovariance(Symmetric(obs_cov), obs_mean, n_man, n_obs) end ############################################################################################ diff --git a/src/observed/data.jl b/src/observed/data.jl index 678f33512..b7bf49505 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -50,7 +50,7 @@ end # error checks function check_arguments_SemObservedData(kwargs...) - # data is a data frame, + # data is a data frame, end @@ -105,7 +105,7 @@ function SemObservedData(; end return SemObservedData(data, - compute_covariance ? Statistics.cov(data) : nothing, + compute_covariance ? Symmetric(cov(data)) : nothing, meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, Float64.(size(data, 2)), Float64.(size(data, 1)), diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 3cc7663c7..02c6b0c1d 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -19,7 +19,7 @@ struct SemObservedMissingPattern{T,S} data::Matrix{T} # non-missing submatrix of data obs_mean::Vector{S} # means of observed vars - obs_cov::Matrix{S} # covariance of observed vars + obs_cov::Symmetric{S, Matrix{S}} # covariance of observed vars end function SemObservedMissingPattern( @@ -44,7 +44,7 @@ function SemObservedMissingPattern( obs_mask, miss_mask, sum(obs_mask), sum(miss_mask), rows, pat_data, - dropdims(pat_mean, dims=1), pat_cov) + dropdims(pat_mean, dims=1), Symmetric(pat_cov)) end n_man(pat::SemObservedMissingPattern) = length(pat.obs_mask) From 12b31ec497885b8a2085b6353791106ffa09075c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 20:30:26 -0700 Subject: [PATCH 112/174] tiny simplification --- src/frontend/fit/fitmeasures/n_obs.jl | 2 +- src/types.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/frontend/fit/fitmeasures/n_obs.jl b/src/frontend/fit/fitmeasures/n_obs.jl index 2da902d7f..782bf002e 100644 --- a/src/frontend/fit/fitmeasures/n_obs.jl +++ b/src/frontend/fit/fitmeasures/n_obs.jl @@ -13,4 +13,4 @@ n_obs(sem_fit::SemFit) = n_obs(sem_fit.model) n_obs(model::AbstractSemSingle) = n_obs(model.observed) -n_obs(model::SemEnsemble) = sum(n_obs.(model.sems)) \ No newline at end of file +n_obs(model::SemEnsemble) = sum(n_obs, model.sems) \ No newline at end of file diff --git a/src/types.jl b/src/types.jl index f7d18f267..d1392e300 100644 --- a/src/types.jl +++ b/src/types.jl @@ -210,7 +210,7 @@ function SemEnsemble(models...; optimizer = SemOptimizerOptim, weights = nothing # default weights if isnothing(weights) - nobs_total = sum(n_obs.(models)) + nobs_total = sum(n_obs, models) weights = [n_obs(model)/nobs_total for model in models] end From c35ae53d0f41ed7b74191d3a1f86cf81b09f57c8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 20:36:06 -0700 Subject: [PATCH 113/174] se_hessian(): rename hessian -> method for clarity --- src/frontend/fit/standard_errors/hessian.jl | 40 ++++++++++---------- src/frontend/specification/ParameterTable.jl | 14 +++---- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 0607a8895..67a38b152 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -1,32 +1,30 @@ """ - se_hessian(semfit::SemFit; hessian = :finitediff) + se_hessian(fit::SemFit; method = :finitediff) -Return hessian based standard errors. +Return hessian-based standard errors. # Arguments -- `hessian`: how to compute the hessian. Options are +- `method`: how to compute the hessian. Options are - `:analytic`: (only if an analytic hessian for the model can be computed) - `:finitediff`: for finite difference approximation """ -function se_hessian(sem_fit::SemFit; hessian = :finitediff) - - c = H_scaling(sem_fit.model) - - if hessian == :analytic - par = solution(sem_fit) - H = zeros(eltype(par), length(par), length(par)) - hessian!(H, sem_fit.model, sem_fit.solution) - elseif hessian == :finitediff - H = FiniteDiff.finite_difference_hessian( - p -> evaluate!(eltype(sem_fit.solution), nothing, nothing, fit.model, p), - sem_fit.solution - ) - elseif hessian == :optimizer - throw(ArgumentError("standard errors from the optimizer hessian are not implemented yet")) - elseif hessian == :expected - throw(ArgumentError("standard errors based on the expected hessian are not implemented yet")) +function se_hessian(fit::SemFit; method = :finitediff) + + c = H_scaling(fit.model) + params = solution(fit) + H = similar(params, (length(params), length(params))) + + if method == :analytic + evaluate!(nothing, nothing, H, fit.model, params) + elseif method == :finitediff + FiniteDiff.finite_difference_hessian!(H, + p -> evaluate!(eltype(H), nothing, nothing, fit.model, p), params) + elseif method == :optimizer + error("Standard errors from the optimizer hessian are not implemented yet") + elseif method == :expected + error("Standard errors based on the expected hessian are not implemented yet") else - throw(ArgumentError("I don't know how to compute `$hessian` standard-errors")) + throw(ArgumentError("Unsupported hessian calculation method :$method")) end invH = c*inv(H) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 759eaa458..7135bcef8 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -331,23 +331,23 @@ end """ update_se_hessian!( partable::AbstractParameterTable, - sem_fit::SemFit; - hessian = :finitediff) + fit::SemFit; + method = :finitediff) Write hessian standard errors computed for `sem_fit` to the `:se` column of `partable` # Arguments -- `hessian::Symbol`: how to compute the hessian, see [se_hessian](@ref) for more information. +- `method::Symbol`: how to compute the hessian, see [se_hessian](@ref) for more information. # Examples """ function update_se_hessian!( partable::AbstractParameterTable, - sem_fit::SemFit; - hessian = :finitediff) - se = se_hessian(sem_fit; hessian = hessian) - return update_partable!(partable, sem_fit, se, :se) + fit::SemFit; + method = :finitediff) + se = se_hessian(fit; method = method) + return update_partable!(partable, fit, se, :se) end """ From 032abfbff8ff6a0a2adde739703b72f439facaef Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 14:20:11 -0700 Subject: [PATCH 114/174] se_hessian!(): optimize calc * explicitly use Cholesky factorization --- src/frontend/fit/standard_errors/hessian.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 67a38b152..1d02efa7f 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -27,10 +27,9 @@ function se_hessian(fit::SemFit; method = :finitediff) throw(ArgumentError("Unsupported hessian calculation method :$method")) end - invH = c*inv(H) - se = sqrt.(diag(invH)) - - return se + H_chol = cholesky!(Symmetric(H)) + H_inv = LinearAlgebra.inv!(H_chol) + return [sqrt(c*H_inv[i]) for i in diagind(H_inv)] end # Addition functions ------------------------------------------------------------- From b5a7ad2db0c37ca6eb2a4a0de6fb32d69a83b5dc Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 19 Mar 2024 20:38:00 -0700 Subject: [PATCH 115/174] H_scaling(): cleanup remove unnecesary arguments --- src/frontend/fit/standard_errors/hessian.jl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/frontend/fit/standard_errors/hessian.jl b/src/frontend/fit/standard_errors/hessian.jl index 1d02efa7f..ba9b79f7a 100644 --- a/src/frontend/fit/standard_errors/hessian.jl +++ b/src/frontend/fit/standard_errors/hessian.jl @@ -33,23 +33,22 @@ function se_hessian(fit::SemFit; method = :finitediff) end # Addition functions ------------------------------------------------------------- -H_scaling(model::AbstractSemSingle) = - H_scaling( - model, - model.observed, - model.imply, - model.optimizer, - model.loss.functions...) - -H_scaling(model, obs, imp, optimizer, lossfun::SemML) = +function H_scaling(model::AbstractSemSingle) + if length(model.loss.functions) > 1 + @warn "Hessian scaling for multiple loss functions is not implemented yet" + end + return H_scaling(model.loss.functions[1], model) +end + +H_scaling(lossfun::SemML, model::AbstractSemSingle) = 2/(n_obs(model)-1) -function H_scaling(model, obs, imp, optimizer, lossfun::SemWLS) +function H_scaling(lossfun::SemWLS, model::AbstractSemSingle) @warn "Standard errors for WLS are only correct if a GLS weight matrix (the default) is used." return 2/(n_obs(model)-1) end -H_scaling(model, obs, imp, optimizer, lossfun::SemFIML) = +H_scaling(lossfun::SemFIML, model::AbstractSemSingle) = 2/n_obs(model) H_scaling(model::SemEnsemble) = From 3d6be597106849d17a87e333cf7341d7faa65d75 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 12:24:05 -0700 Subject: [PATCH 116/174] EM: optimizations --- src/observed/EM.jl | 110 +++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index f4f6510cd..0c25f0ea2 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -28,8 +28,8 @@ function em_mvn( max_iter_em = 100, rtol_em = 1e-4, kwargs...) - - n_obs, n_man = observed.n_obs, Int(observed.n_man) + + n_man = SEM.n_man(observed) # preallocate stuff? 𝔼x_pre = zeros(n_man) @@ -38,18 +38,18 @@ function em_mvn( ### precompute for full cases fullpat = observed.patterns[1] if nmissed_vars(fullpat) == 0 - for row in eachrow(fullpat.data) - 𝔼x_pre += row; - 𝔼xxᵀ_pre += row*row'; - end + sum!(reshape(𝔼x_pre, 1, n_man), fullpat.data) + mul!(𝔼xxᵀ_pre, fullpat.data', fullpat.data) + else + @warn "No full cases pattern found" end - + # ess = 𝔼x, 𝔼xxᵀ, ismissing, missingRows, n_obs # estepFn = (em_model, data) -> estep(em_model, data, EXsum, EXXsum, ismissing, missingRows, n_obs) # initialize em_model = start_em(observed; kwargs...) - em_model_prev = EmMVNModel(zeros(n_man, n_man), zeros(n_man), false) + em_model_prev = EmMVNModel(zeros(n_man, n_man), zeros(n_man), false) iter = 1 done = false 𝔼x = zeros(n_man) @@ -57,21 +57,22 @@ function em_mvn( while !done - em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xxᵀ_pre) - em_mvn_Mstep!(em_model, n_obs, 𝔼x, 𝔼xxᵀ) + step!(em_model, observed, 𝔼x, 𝔼xxᵀ, 𝔼x_pre, 𝔼xxᵀ_pre) if iter > max_iter_em done = true - @warn "EM Algorithm for MVN missing data did not converge. Likelihood for FIML is not interpretable. + @warn "EM Algorithm for MVN missing data did not converge. Likelihood for FIML is not interpretable. Maybe try passing different starting values via 'start_em = ...' " elseif iter > 1 # done = isapprox(ll, ll_prev; rtol = rtol) - done = isapprox(em_model_prev.μ, em_model.μ; rtol = rtol_em) & isapprox(em_model_prev.Σ, em_model.Σ; rtol = rtol_em) + done = isapprox(em_model_prev.μ, em_model.μ; rtol = rtol_em) && + isapprox(em_model_prev.Σ, em_model.Σ; rtol = rtol_em) end # print("$iter \n") - iter = iter + 1 - em_model_prev.μ, em_model_prev.Σ = em_model.μ, em_model.Σ + iter += 1 + copyto!(em_model_prev.μ, em_model.μ) + copyto!(em_model_prev.Σ, em_model.Σ) end @@ -81,18 +82,17 @@ function em_mvn( observed.em_model.fitted = true return nothing - -end - -# E and M step ----------------------------------------------------------------------------- -function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xxᵀ_pre) +end - 𝔼x .= 0.0 - 𝔼xxᵀ .= 0.0 +# E and M steps ----------------------------------------------------------------------------- - 𝔼xᵢ = copy(𝔼x) - 𝔼xxᵀᵢ = copy(𝔼xxᵀ) +# update em_model +function step!(em_model::EmMVNModel, observed::SemObserved, + 𝔼x, 𝔼xxᵀ, 𝔼x_pre, 𝔼xxᵀ_pre) + # E step, update 𝔼x and 𝔼xxᵀ + fill!(𝔼x, 0) + fill!(𝔼xxᵀ, 0) μ = em_model.μ Σ = em_model.Σ @@ -106,47 +106,51 @@ function em_mvn_Estep!(𝔼x, 𝔼xxᵀ, em_model, observed, 𝔼x_pre, 𝔼xx o = pat.obs_mask # precompute for pattern - Σoo = Σ[o, o] + Σoo_chol = cholesky(Symmetric(Σ[o, o])) Σuo = Σ[u, o] μu = μ[u] μo = μ[o] - V = Σ[u, u] - Σuo * (Σoo \ Σ[o, u]) + 𝔼xu = fill!(similar(μu), 0) + 𝔼xo = fill!(similar(μo), 0) + 𝔼xᵢu = similar(μu) + + 𝔼xxᵀuo = fill!(similar(Σuo), 0) + 𝔼xxᵀuu = n_obs(pat) * (Σ[u, u] - Σuo * (Σoo_chol \ Σuo')) # loop trough data - for rowdata in eachrow(pat.data) - m = μu + Σuo * ( Σoo \ (rowdata-μo) ) - - 𝔼xᵢ[u] = m - 𝔼xᵢ[o] = rowdata - 𝔼xxᵀᵢ[u, u] = 𝔼xᵢ[u] * 𝔼xᵢ[u]' + V - 𝔼xxᵀᵢ[o, o] = 𝔼xᵢ[o] * 𝔼xᵢ[o]' - 𝔼xxᵀᵢ[o, u] = 𝔼xᵢ[o] * 𝔼xᵢ[u]' - 𝔼xxᵀᵢ[u, o] = 𝔼xᵢ[u] * 𝔼xᵢ[o]' - - 𝔼x .+= 𝔼xᵢ - 𝔼xxᵀ .+= 𝔼xxᵀᵢ + @inbounds for rowdata in eachrow(pat.data) + mul!(𝔼xᵢu, Σuo, Σoo_chol \ (rowdata-μo)) + 𝔼xᵢu .+= μu + mul!(𝔼xxᵀuu, 𝔼xᵢu, 𝔼xᵢu', 1, 1) + mul!(𝔼xxᵀuo, 𝔼xᵢu, rowdata', 1, 1) + 𝔼xu .+= 𝔼xᵢu + 𝔼xo .+= rowdata end + 𝔼xxᵀ[o,o] .+= pat.data' * pat.data + 𝔼xxᵀ[u,o] .+= 𝔼xxᵀuo + 𝔼xxᵀ[o,u] .+= 𝔼xxᵀuo' + 𝔼xxᵀ[u,u] .+= 𝔼xxᵀuu + + 𝔼x[o] .+= 𝔼xo + 𝔼x[u] .+= 𝔼xu end 𝔼x .+= 𝔼x_pre 𝔼xxᵀ .+= 𝔼xxᵀ_pre -end - -function em_mvn_Mstep!(em_model, n_obs, 𝔼x, 𝔼xxᵀ) - - em_model.μ = 𝔼x/n_obs; - Σ = Symmetric(𝔼xxᵀ/n_obs - em_model.μ*em_model.μ') - + # M step, update em_model + em_model.μ .= 𝔼x ./ n_obs(observed) + em_model.Σ .= 𝔼xxᵀ ./ n_obs(observed) + mul!(em_model.Σ, em_model.μ, em_model.μ', -1, 1) + + #Σ = em_model.Σ # ridge Σ # while !isposdef(Σ) # Σ += 0.5I # end - em_model.Σ = Σ - # diagonalization #if !isposdef(Σ) # print("Matrix not positive definite") @@ -156,7 +160,7 @@ function em_mvn_Mstep!(em_model, n_obs, 𝔼x, 𝔼xxᵀ) # em_model.Σ = Σ #end - return nothing + return em_model end # generate starting values ----------------------------------------------------------------- @@ -167,11 +171,11 @@ function start_em_observed(observed::SemObservedMissing; kwargs...) fullpat = observed.patterns[1] if (nmissed_vars(fullpat) == 0) && (n_obs(fullpat) > 1) μ = copy(fullpat.obs_mean) - Σ = copy(Symmetric(fullpat.obs_cov)) + Σ = copy(fullpat.obs_cov) if !isposdef(Σ) - Σ = Matrix(Diagonal(Σ)) + Σ = Diagonal(Σ) end - return EmMVNModel(Σ, μ, false) + return EmMVNModel(convert(Matrix, Σ), μ, false) else return start_em_simple(observed, kwargs...) end @@ -180,9 +184,9 @@ end # use μ = O and Σ = I function start_em_simple(observed::SemObservedMissing; kwargs...) - n_man = Int(observed.n_man) - μ = zeros(n_man) - Σ = rand(n_man, n_man); Σ = Σ*Σ' + μ = zeros(n_man(observed)) + Σ = rand(n_man(observed), n_man(observed)) + Σ = Σ*Σ' # Σ = Matrix(1.0I, n_man, n_man) return EmMVNModel(Σ, μ, false) end From 992bfda046ddff213e11b274bc720d90539fc9fd Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 07:58:04 -0700 Subject: [PATCH 117/174] SemObsMissing: remove outdated docstring --- src/observed/missing.jl | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 02c6b0c1d..d13a0d938 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -75,16 +75,7 @@ For observed data with missing values. - `n_man(::SemObservedMissing)` -> number of manifest variables - `get_data(::SemObservedMissing)` -> observed data -- `data_rowwise(::SemObservedMissing)` -> observed data as vector per observation, with missing values deleted - -- `patterns(::SemObservedMissing)` -> indices of non-missing variables per missing patterns -- `patterns_not(::SemObservedMissing)` -> indices of missing variables per missing pattern -- `rows(::SemObservedMissing)` -> row indices of observed data points that belong to each pattern -- `pattern_n_obs(::SemObservedMissing)` -> number of data points per pattern -- `pattern_nvar_obs(::SemObservedMissing)` -> number of non-missing observed variables per pattern -- `obs_mean(::SemObservedMissing)` -> observed mean per pattern -- `obs_cov(::SemObservedMissing)` -> observed covariance per pattern -- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via optimization maximization +- `em_model(::SemObservedMissing)` -> `EmMVNModel` that contains the covariance matrix and mean vector found via expectation maximization ## Implementation Subtype of `SemObserved` From 5bf1369aa0a065e1ce6e67e69c449680c31cfe88 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 07:59:15 -0700 Subject: [PATCH 118/174] SemObsData: remove rowwise * it is unused * if ever rowwise access would be required, it could be done with eachrow(data) without allocation --- src/observed/data.jl | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/observed/data.jl b/src/observed/data.jl index b7bf49505..57df79662 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -24,7 +24,6 @@ For observed data without missings. - `get_data(::SemObservedData)` -> observed data - `obs_cov(::SemObservedData)` -> observed.obs_cov - `obs_mean(::SemObservedData)` -> observed.obs_mean -- `data_rowwise(::SemObservedData)` -> observed data, stored as vectors per observation ## Implementation Subtype of `SemObserved` @@ -37,15 +36,13 @@ use this if you are sure your observed data is in the right format. ## Additional keyword arguments: - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object - `compute_covariance::Bool ) = true`: should the covariance of `data` be computed and stored? -- `rowwise::Bool = false`: should the data be stored also as vectors per observation """ -struct SemObservedData{A, B, C, D, O, R} <: SemObserved +struct SemObservedData{A, B, C, D, O} <: SemObserved data::A obs_cov::B obs_mean::C n_man::D n_obs::O - data_rowwise::R end # error checks @@ -65,8 +62,6 @@ function SemObservedData(; meanstructure = false, compute_covariance = true, - rowwise = false, - kwargs...) if isnothing(spec_colnames) && !isnothing(specification) @@ -108,8 +103,7 @@ function SemObservedData(; compute_covariance ? Symmetric(cov(data)) : nothing, meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, Float64.(size(data, 2)), - Float64.(size(data, 1)), - rowwise ? [data[i, :] for i in axes(data, 1)] : nothing) + Float64.(size(data, 1))) end ############################################################################################ @@ -125,7 +119,6 @@ n_man(observed::SemObservedData) = observed.n_man obs_cov(observed::SemObservedData) = observed.obs_cov obs_mean(observed::SemObservedData) = observed.obs_mean -data_rowwise(observed::SemObservedData) = observed.data_rowwise ############################################################################################ ### Additional functions From 2e530570bd675b96e93573b3988deba4892683f6 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 11:25:21 -0700 Subject: [PATCH 119/174] cleanup data columns reordering define a single source_to_dest_perm() function --- src/StructuralEquationModels.jl | 2 +- src/observed/covariance.jl | 31 +++---------------------------- src/observed/data.jl | 15 +++++++-------- src/observed/missing.jl | 2 +- 4 files changed, 12 insertions(+), 38 deletions(-) diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index baf890737..d5fbeefeb 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -25,8 +25,8 @@ include("frontend/fit/summary.jl") # pretty printing include("frontend/pretty_printing.jl") # observed -include("observed/covariance.jl") include("observed/data.jl") +include("observed/covariance.jl") include("observed/missing.jl") include("observed/EM.jl") # constructor diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 6b99d6d4c..246d4b469 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -75,8 +75,9 @@ function SemObservedCovariance(; end if !isnothing(spec_colnames) - obs_cov = reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) - isnothing(obs_mean) || (obs_mean = reorder_obs_mean(obs_mean, spec_colnames, obs_colnames)) + obs2spec_perm = source_to_dest_perm(obs_colnames, spec_colnames) + obs_cov = obs_cov[obs2spec_perm, obs2spec_perm] + isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end n_man = Float64(size(obs_cov, 1)) @@ -98,29 +99,3 @@ n_man(observed::SemObservedCovariance) = observed.n_man obs_cov(observed::SemObservedCovariance) = observed.obs_cov obs_mean(observed::SemObservedCovariance) = observed.obs_mean -############################################################################################ -### Additional functions -############################################################################################ - -# reorder covariance matrices -------------------------------------------------------------- -function reorder_obs_cov(obs_cov, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return obs_cov - else - new_position = [findfirst(==(x), obs_colnames) for x in spec_colnames] - obs_cov = obs_cov[new_position, new_position] - return obs_cov - end -end - -# reorder means ---------------------------------------------------------------------------- - -function reorder_obs_mean(obs_mean, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return obs_mean - else - new_position = [findfirst(==(x), obs_colnames) for x in spec_colnames] - obs_mean = obs_mean[new_position] - return obs_mean - end -end diff --git a/src/observed/data.jl b/src/observed/data.jl index 57df79662..1fc9b510b 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -91,7 +91,7 @@ function SemObservedData(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = reorder_data(data, spec_colnames, obs_colnames) + data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] end end @@ -124,13 +124,12 @@ obs_mean(observed::SemObservedData) = observed.obs_mean ### Additional functions ############################################################################################ -# reorder data ----------------------------------------------------------------------------- -function reorder_data(data::AbstractArray, spec_colnames, obs_colnames) - if spec_colnames == obs_colnames - return data +# permutation that subsets and reorders source to matches the destination order ------------ +function source_to_dest_perm(src::AbstractVector, dest::AbstractVector) + if dest == src # exact match + return eachindex(dest) else - obs_positions = Dict(col => i for (i, col) in enumerate(obs_colnames)) - new_positions = [obs_positions[col] for col in spec_colnames] - return data[:, new_positions] + src_inds = Dict(el => i for (i, el) in enumerate(src)) + return [src_inds[el] for el in dest] end end \ No newline at end of file diff --git a/src/observed/missing.jl b/src/observed/missing.jl index d13a0d938..8c66ffda8 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -140,7 +140,7 @@ function SemObservedMissing(; throw(ArgumentError("please specify `obs_colnames` as a vector of Symbols")) end - data = reorder_data(data, spec_colnames, obs_colnames) + data = data[:, source_to_dest_perm(obs_colnames, spec_colnames)] end end From e9ab0c0b3745e29720889b6079a08f954b82252d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 09:56:32 -0700 Subject: [PATCH 120/174] n_obs/man(data): restrict to integer don't allow missing n_obs --- src/imply/RAM/symbolic.jl | 2 +- src/observed/covariance.jl | 12 ++++---- src/observed/data.jl | 9 +++--- .../political_democracy/constructor.jl | 4 +-- test/unit_tests/data_input_formats.jl | 29 ++++++++++--------- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index 7e46f2088..d32157c19 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -213,7 +213,7 @@ params(imply::RAMSymbolic) = params(imply.ram_matrices) nparams(imply::RAMSymbolic) = nparams(imply.ram_matrices) function update_observed(imply::RAMSymbolic, observed::SemObserved; kwargs...) - if Int(n_man(observed)) == size(imply.Σ, 1) + if n_man(observed) == size(imply.Σ, 1) return imply else return RAMSymbolic(;observed = observed, kwargs...) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 246d4b469..6dd637b6d 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -39,11 +39,11 @@ use this if you are sure your covariance matrix is in the right format. ## Additional keyword arguments: - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object """ -struct SemObservedCovariance{B, C, D, O} <: SemObserved +struct SemObservedCovariance{B, C} <: SemObserved obs_cov::B obs_mean::C - n_man::D - n_obs::O + n_man::Int + n_obs::Int end function SemObservedCovariance(; @@ -56,7 +56,7 @@ function SemObservedCovariance(; obs_mean::Union{AbstractVector, Nothing} = nothing, meanstructure::Bool = false, - n_obs::Union{Number, Nothing} = nothing, + n_obs::Integer, kwargs...) @@ -80,9 +80,7 @@ function SemObservedCovariance(; isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end - n_man = Float64(size(obs_cov, 1)) - - return SemObservedCovariance(Symmetric(obs_cov), obs_mean, n_man, n_obs) + return SemObservedCovariance(Symmetric(obs_cov), obs_mean, size(obs_cov, 1), n_obs) end ############################################################################################ diff --git a/src/observed/data.jl b/src/observed/data.jl index 1fc9b510b..bcc31aed4 100644 --- a/src/observed/data.jl +++ b/src/observed/data.jl @@ -37,12 +37,12 @@ use this if you are sure your observed data is in the right format. - `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object - `compute_covariance::Bool ) = true`: should the covariance of `data` be computed and stored? """ -struct SemObservedData{A, B, C, D, O} <: SemObserved +struct SemObservedData{A, B, C} <: SemObserved data::A obs_cov::B obs_mean::C - n_man::D - n_obs::O + n_man::Int + n_obs::Int end # error checks @@ -102,8 +102,7 @@ function SemObservedData(; return SemObservedData(data, compute_covariance ? Symmetric(cov(data)) : nothing, meanstructure ? vec(Statistics.mean(data, dims = 1)) : nothing, - Float64.(size(data, 2)), - Float64.(size(data, 1))) + size(data, 2), size(data, 1)) end ############################################################################################ diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 5a5219522..095db75f4 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -17,7 +17,7 @@ model_ml_cov = Sem( obs_cov = cov(Matrix(dat)), obs_colnames = Symbol.(names(dat)), optimizer = semoptimizer, - n_obs = 75.0 + n_obs = 75 ) model_ls_sym = Sem( @@ -221,7 +221,7 @@ model_ml_cov = Sem( obs_colnames = Symbol.(names(dat)), meanstructure = true, optimizer = semoptimizer, - n_obs = 75.0 + n_obs = 75 ) model_ml_sym = Sem( diff --git a/test/unit_tests/data_input_formats.jl b/test/unit_tests/data_input_formats.jl index 226132a42..281a30123 100644 --- a/test/unit_tests/data_input_formats.jl +++ b/test/unit_tests/data_input_formats.jl @@ -200,30 +200,31 @@ end # errors @test_throws ArgumentError("observed means were passed, but `meanstructure = false`") begin - SemObservedCovariance(specification = nothing, obs_cov = dat_cov, obs_mean = dat_mean) + SemObservedCovariance(specification = nothing, obs_cov = dat_cov, obs_mean = dat_mean, n_obs = 75) end #@test_throws UndefKeywordError(:specification) SemObservedCovariance(obs_cov = dat_cov) @test_throws ArgumentError("no `obs_colnames` were specified") begin - SemObservedCovariance(specification = spec, obs_cov = dat_cov) + SemObservedCovariance(specification = spec, obs_cov = dat_cov, n_obs = 75) end @test_throws TypeError begin - SemObservedCovariance(specification = spec, obs_cov = dat_cov, obs_colnames = names(dat)) + SemObservedCovariance(specification = spec, obs_cov = dat_cov, obs_colnames = names(dat), n_obs = 75) end # should work observed = SemObservedCovariance( specification = spec, obs_cov = dat_cov, - obs_colnames = obs_colnames = Symbol.(names(dat)) + obs_colnames = obs_colnames = Symbol.(names(dat)), + n_obs = 75 ) observed_nospec = SemObservedCovariance( specification = nothing, obs_cov = dat_cov, - n_obs = 75.0 + n_obs = 75 ) @@ -231,8 +232,8 @@ all_equal_cov = (obs_cov(observed) == obs_cov(observed_nospec)) @testset "unit tests | SemObservedCovariance | input formats" begin @test all_equal_cov - @test isnothing(n_obs(observed)) - @test n_obs(observed_nospec) == 75.0 + @test n_obs(observed) == 75 + @test n_obs(observed_nospec) == 75 end @@ -248,7 +249,8 @@ shuffle_dat_cov = Statistics.cov(shuffle_dat_matrix) observed_shuffle = SemObservedCovariance( specification = spec, obs_cov = shuffle_dat_cov, - obs_colnames = shuffle_names + obs_colnames = shuffle_names, + n_obs = 75 ) all_equal_cov_suffled = (obs_cov(observed) ≈ obs_cov(observed_shuffle)) @@ -261,7 +263,7 @@ end # errors @test_throws ArgumentError("`meanstructure = true`, but no observed means were passed") begin - SemObservedCovariance(specification = spec, obs_cov = dat_cov, meanstructure = true) + SemObservedCovariance(specification = spec, obs_cov = dat_cov, meanstructure = true, n_obs=75) end #@test_throws UndefKeywordError SemObservedCovariance(data = dat_matrix, meanstructure = true) @@ -273,7 +275,8 @@ end specification = spec, obs_cov = dat_cov, obs_colnames = Symbol.(names(dat)), - meanstructure = true + meanstructure = true, + n_obs = 75 ) end @@ -283,7 +286,7 @@ observed = SemObservedCovariance( obs_cov = dat_cov, obs_mean = dat_mean, obs_colnames = Symbol.(names(dat)), - n_obs = 75.0, + n_obs = 75, meanstructure = true ) @@ -292,7 +295,7 @@ observed_nospec = SemObservedCovariance( obs_cov = dat_cov, obs_mean = dat_mean, meanstructure = true, - n_obs = 75.0 + n_obs = 75 ) all_equal_mean = (obs_mean(observed) == obs_mean(observed_nospec)) @@ -318,7 +321,7 @@ observed_shuffle = SemObservedCovariance( obs_cov = shuffle_dat_cov, obs_mean = shuffle_dat_mean, obs_colnames = shuffle_names, - n_obs = 75.0, + n_obs = 75, meanstructure = true ) From 34b611d322081c79be7133733cd1ba6bf071f86a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 10:31:33 -0700 Subject: [PATCH 121/174] SemObsCov is a type alias for SemObsData reduces code duplication --- src/observed/covariance.jl | 64 ++------------------------------------ 1 file changed, 3 insertions(+), 61 deletions(-) diff --git a/src/observed/covariance.jl b/src/observed/covariance.jl index 6dd637b6d..c4e94c2f0 100644 --- a/src/observed/covariance.jl +++ b/src/observed/covariance.jl @@ -1,50 +1,7 @@ """ -For observed covariance matrices and means. - -# Constructor - - SemObservedCovariance(; - specification, - obs_cov, - obs_colnames = nothing, - meanstructure = false, - obs_mean = nothing, - n_obs = nothing, - kwargs...) - -# Arguments -- `specification`: either a `RAMMatrices` or `ParameterTable` object (1) -- `obs_cov`: observed covariance matrix -- `obs_colnames::Vector{Symbol}`: column names of the covariance matrix -- `meanstructure::Bool`: does the model have a meanstructure? -- `obs_mean`: observed mean vector -- `n_obs::Number`: number of observed data points (necessary for fit statistics) - -# Extended help -## Interfaces -- `n_obs(::SemObservedCovariance)` -> number of observed data points -- `n_man(::SemObservedCovariance)` -> number of manifest variables - -- `obs_cov(::SemObservedCovariance)` -> observed covariance matrix -- `obs_mean(::SemObservedCovariance)` -> observed means - -## Implementation -Subtype of `SemObserved` - -## Remarks -(1) the `specification` argument can also be `nothing`, but this turns of checking whether -the observed data/covariance columns are in the correct order! As a result, you should only -use this if you are sure your covariance matrix is in the right format. - -## Additional keyword arguments: -- `spec_colnames::Vector{Symbol} = nothing`: overwrites column names of the specification object +Type alias for [SemObservedData](@ref) with no data, but with mean and covariance. """ -struct SemObservedCovariance{B, C} <: SemObserved - obs_cov::B - obs_mean::C - n_man::Int - n_obs::Int -end +SemObservedCovariance{B, C} = SemObservedData{Nothing, B, C} function SemObservedCovariance(; specification::Union{SemSpecification, Nothing} = nothing, @@ -80,20 +37,5 @@ function SemObservedCovariance(; isnothing(obs_mean) || (obs_mean = obs_mean[obs2spec_perm]) end - return SemObservedCovariance(Symmetric(obs_cov), obs_mean, size(obs_cov, 1), n_obs) + return SemObservedData(nothing, Symmetric(obs_cov), obs_mean, size(obs_cov, 1), n_obs) end - -############################################################################################ -### Recommended methods -############################################################################################ - -n_obs(observed::SemObservedCovariance) = observed.n_obs -n_man(observed::SemObservedCovariance) = observed.n_man - -############################################################################################ -### additional methods -############################################################################################ - -obs_cov(observed::SemObservedCovariance) = observed.obs_cov -obs_mean(observed::SemObservedCovariance) = observed.obs_mean - From 68c22c54ab85efd2d9c3d7daf4dca2c2874176a8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 16:40:51 -0700 Subject: [PATCH 122/174] DataFrame(EnsParTable) --- .../specification/EnsembleParameterTable.jl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index fa6a9ca64..bfc95264a 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -62,13 +62,16 @@ function Base.convert(::Type{Dict{K, RAMMatrices}}, ) end -#= function DataFrame( - partable::ParameterTable; - columns = nothing) - if isnothing(columns) columns = keys(partable.columns) end - out = DataFrame([key => partable.columns[key] for key in columns]) - return DataFrame(out) -end =# +function DataFrames.DataFrame( + partables::EnsembleParameterTable; + columns::Union{AbstractVector{Symbol}, Nothing} = nothing +) + mapreduce(vcat, pairs(partables.tables)) do (key, partable) + df = DataFrame(partable; columns = columns) + df[!, :group] .= key + return df + end +end ############################################################################################ ### Pretty Printing From 9dff90f3b976357aa4f386f6ffa1b046a79afe3d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 16:41:48 -0700 Subject: [PATCH 123/174] test: use proper partable --- test/examples/multigroup/build_models.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/examples/multigroup/build_models.jl b/test/examples/multigroup/build_models.jl index 2ffd1c308..e24238bab 100644 --- a/test/examples/multigroup/build_models.jl +++ b/test/examples/multigroup/build_models.jl @@ -87,7 +87,7 @@ grad_fd = FiniteDiff.finite_difference_gradient(x -> SEM.objective!(model_ml_mul solution = sem_fit(model_ml_multigroup) update_estimate!(partable_s, solution) test_estimates( - partable, + partable_s, solution_lav[:parameter_estimates_ml]; atol = 1e-4, lav_groups = Dict(:Pasteur => 1, :Grant_White => 2)) end From 5e7d3bb8de4492d145d5bd0d7bf2fb6e117ba92c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 16:42:29 -0700 Subject: [PATCH 124/174] start_simple(SemEnsemble): simplify --- .../start_val/start_simple.jl | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/additional_functions/start_val/start_simple.jl b/src/additional_functions/start_val/start_simple.jl index fee6cc989..0b43e9cff 100644 --- a/src/additional_functions/start_val/start_simple.jl +++ b/src/additional_functions/start_val/start_simple.jl @@ -32,22 +32,18 @@ end # Ensemble Models -------------------------------------------------------------------------- function start_simple(model::SemEnsemble; kwargs...) - start_vals = [] + start_vals = fill(0.0, nparams(model)) for sem in model.sems - push!(start_vals, start_simple(sem; kwargs...)) - end - - has_start_val = [.!iszero.(start_val) for start_val in start_vals] - - start_val = similar(start_vals[1]) - start_val .= 0.0 - - for (j, indices) in enumerate(has_start_val) - start_val[indices] .= start_vals[j][indices] + sem_start_vals = start_simple(sem; kwargs...) + for (i, val) in enumerate(sem_start_vals) + if !iszero(val) + start_vals[i] = val + end + end end - return start_val + return start_vals end From 74449e80b62fc9a098a5995c55ac9e10b83aee55 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:35:50 -0700 Subject: [PATCH 125/174] SemOptOptim: remove redundant sem_fit() by dispatching over optimizer --- src/optimizer/documentation.jl | 9 ++++++++- src/optimizer/optim.jl | 29 ++++++----------------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index ff9d9977f..7c17e6ce2 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -19,4 +19,11 @@ sem_fit( start_covariances_latent = 0.5) ``` """ -function sem_fit end \ No newline at end of file +function sem_fit end + +# dispatch on optimizer +sem_fit(model::AbstractSem; kwargs...) = sem_fit(model.optimizer, model; kwargs...) + +# fallback method +sem_fit(optimizer::SemOptimizer, model::AbstractSem; kwargs...) = + error("Optimizer $(optimizer) support not implemented.") diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 176b53edd..7d79668e3 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -44,29 +44,12 @@ n_iterations(res::Optim.MultivariateOptimizationResults) = Optim.iterations(res) convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res) function sem_fit( - model::AbstractSemSingle{O, I, L, D}; - start_val = start_val, - kwargs...) where {O, I, L, D <: SemOptimizerOptim} - - if !isa(start_val, Vector) - start_val = start_val(model; kwargs...) - end - - result = Optim.optimize( - Optim.only_fgh!((F, G, H, par) -> sem_wrap_optim(par, F, G, H, model)), - start_val, - model.optimizer.algorithm, - model.optimizer.options) - return SemFit(result, model, start_val) - -end + optim::SemOptimizerOptim, + model::AbstractSem; + start_val = start_val, + kwargs...) -function sem_fit( - model::SemEnsemble{N, T , V, D, S}; - start_val = start_val, - kwargs...) where {N, T, V, D <: SemOptimizerOptim, S} - - if !isa(start_val, Vector) + if !isa(start_val, AbstractVector) start_val = start_val(model; kwargs...) end @@ -77,4 +60,4 @@ function sem_fit( model.optimizer.options) return SemFit(result, model, start_val) -end \ No newline at end of file +end From e398e9d01884cd8fb684ea1718f0d2e04a98e5ef Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:37:05 -0700 Subject: [PATCH 126/174] SemOptNLopt: remove redundant sem_fit() by dispatching over optimizer --- src/optimizer/NLopt.jl | 36 +++++------------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/src/optimizer/NLopt.jl b/src/optimizer/NLopt.jl index 5e46b73d4..94f84e6f0 100644 --- a/src/optimizer/NLopt.jl +++ b/src/optimizer/NLopt.jl @@ -33,39 +33,13 @@ function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) end # sem_fit method -function sem_fit(model::Sem{O, I, L, D}; start_val = start_val, kwargs...) where {O, I, L, D <: SemOptimizerNLopt} +function sem_fit( + optimizer::SemOptimizerNLopt, + model::AbstractSem; + start_val = start_val, kwargs...) # starting values - if !isa(start_val, Vector) - start_val = start_val(model; kwargs...) - end - - # construct the NLopt problem - opt = construct_NLopt_problem( - model.optimizer.algorithm, - model.optimizer.options, - length(start_val)) - set_NLopt_constraints!(opt, model.optimizer) - opt.min_objective = (par, G) -> sem_wrap_nlopt(par, G, model) - - if !isnothing(model.optimizer.local_algorithm) - opt_local = construct_NLopt_problem( - model.optimizer.local_algorithm, - model.optimizer.local_options, - length(start_val)) - opt.local_optimizer = opt_local - end - - # fit - result = NLopt.optimize(opt, start_val) - - return SemFit_NLopt(result, model, start_val, opt) -end - -function sem_fit(model::SemEnsemble{N, T , V, D, S}; start_val = start_val, kwargs...) where {N, T, V, D <: SemOptimizerNLopt, S} - - # starting values - if !isa(start_val, Vector) + if !isa(start_val, AbstractVector) start_val = start_val(model; kwargs...) end From 98bf2297e0875dd2666ede42860c3af7091a329c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:37:45 -0700 Subject: [PATCH 127/174] SemOptOptim: use evaluate!() directly no wrapper required --- src/optimizer/optim.jl | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index 7d79668e3..f855836c6 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -1,30 +1,4 @@ ## connect to Optim.jl as backend -function sem_wrap_optim(par, F, G, H, model::AbstractSem) - if !isnothing(F) - if !isnothing(G) - if !isnothing(H) - return objective_gradient_hessian!(G, H, model, par) - else - return objective_gradient!(G, model, par) - end - else - if !isnothing(H) - return objective_hessian!(H, model, par) - else - return objective!(model, par) - end - end - else - if !isnothing(G) - if !isnothing(H) - gradient_hessian!(G, H, model, par) - else - gradient!(G, model, par) - end - end - end - return nothing -end function SemFit( optimization_result::Optim.MultivariateOptimizationResults, @@ -54,7 +28,7 @@ function sem_fit( end result = Optim.optimize( - Optim.only_fgh!((F, G, H, par) -> sem_wrap_optim(par, F, G, H, model)), + Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), start_val, model.optimizer.algorithm, model.optimizer.options) From 456251486d370d29349e103aa8cbc862989df0c4 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:38:30 -0700 Subject: [PATCH 128/174] SemOptNLopt: use evaluate!() directly --- src/optimizer/NLopt.jl | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/optimizer/NLopt.jl b/src/optimizer/NLopt.jl index 94f84e6f0..dee309d46 100644 --- a/src/optimizer/NLopt.jl +++ b/src/optimizer/NLopt.jl @@ -2,16 +2,6 @@ ### connect to NLopt.jl as backend ############################################################################################ -# wrapper to define the objective -function sem_wrap_nlopt(par, G, model::AbstractSem) - need_gradient = length(G) != 0 - if need_gradient - return objective_gradient!(G, model, par) - else - return objective!(model, par) - end -end - mutable struct NLoptResult result problem @@ -49,7 +39,7 @@ function sem_fit( model.optimizer.options, length(start_val)) set_NLopt_constraints!(opt, model.optimizer) - opt.min_objective = (par, G) -> sem_wrap_nlopt(par, G, model) + opt.min_objective = (par, G) -> evaluate!(eltype(par), !isnothing(G) && !isempty(G) ? G : nothing, nothing, model, par) if !isnothing(model.optimizer.local_algorithm) opt_local = construct_NLopt_problem( From b3d3c2c0174270e3e23875515df6f2f1f0e45410 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:50:13 -0700 Subject: [PATCH 129/174] remove unused parameters.jl superseded by ParamsArray --- src/StructuralEquationModels.jl | 3 +- src/additional_functions/parameters.jl | 144 ------------------------- 2 files changed, 1 insertion(+), 146 deletions(-) delete mode 100644 src/additional_functions/parameters.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index d5fbeefeb..c3af1cc26 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -52,7 +52,6 @@ include("optimizer/optim.jl") include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") -#include("additional_functions/parameters.jl") include("additional_functions/start_val/start_val.jl") include("additional_functions/start_val/start_fabin3.jl") include("additional_functions/start_val/start_partable.jl") @@ -105,7 +104,7 @@ export AbstractSem, ParameterTable, EnsembleParameterTable, update_partable!, update_estimate!, update_start!, update_se_hessian!, Fixed, fixed, Start, start, Label, label, sort_vars!, sort_vars, - RAMMatrices, + RAMMatrices, RAMMatrices!, params, nparams, fit_measures, diff --git a/src/additional_functions/parameters.jl b/src/additional_functions/parameters.jl deleted file mode 100644 index 67fb37c72..000000000 --- a/src/additional_functions/parameters.jl +++ /dev/null @@ -1,144 +0,0 @@ -# fill A, S, and M matrices with the parameter values according to the parameters map -function fill_A_S_M!( - A::AbstractMatrix, S::AbstractMatrix, - M::Union{AbstractVector, Nothing}, - A_indices::AbstractArrayParamsMap, - S_indices::AbstractArrayParamsMap, - M_indices::Union{AbstractArrayParamsMap, Nothing}, - parameters::AbstractVector -) - - @inbounds for (iA, iS, par) in zip(A_indices, S_indices, parameters) - for index_A in iA - A[index_A] = par - end - - for index_S in iS - S[index_S] = par - end - end - - if !isnothing(M) - @inbounds for (iM, par) in zip(M_indices, parameters) - for index_M in iM - M[index_M] = par - end - end - end - -end - -# build the map from the index of the parameter to the indices of this parameter -# occurences in the given array -function array_parameters_map(parameters::AbstractVector, M::AbstractArray) - params_index = Dict(param => i for (i, param) in enumerate(parameters)) - T = Base.eltype(eachindex(M)) - res = [Vector{T}() for _ in eachindex(parameters)] - for (i, val) in pairs(M) - par_ind = get(params_index, val, nothing) - if !isnothing(par_ind) - push!(res[par_ind], i) - end - end - return res -end - -# build the map of parameter index to the linear indices of its occurences in M -# returns ArrayParamsMap object -array_parameters_map_linear(parameters::AbstractVector, M::AbstractArray) = - array_parameters_map(parameters, vec(M)) - -function eachindex_lower(M; linear_indices = false, kwargs...) - - indices = CartesianIndices(M) - indices = filter(x -> (x[1] >= x[2]), indices) - - if linear_indices - indices = cartesian2linear(indices, M) - end - - return indices - -end - -function cartesian2linear(ind_cart, dims) - ind_lin = LinearIndices(dims)[ind_cart] - return ind_lin -end - -function linear2cartesian(ind_lin, dims) - ind_cart = CartesianIndices(dims)[ind_lin] - return ind_cart -end - -function set_constants!(M, M_pre) - - for index in eachindex(M) - - δ = tryparse(Float64, string(M[index])) - - if !iszero(M[index]) & (δ !== nothing) - M_pre[index] = δ - end - - end - -end - -function check_constants(M) - - for index in eachindex(M) - - δ = tryparse(Float64, string(M[index])) - - if !iszero(M[index]) & (δ !== nothing) - return true - end - - end - - return false - -end - - -# construct length(M)×length(parameters) sparse matrix of 1s at the positions, -# where the corresponding parameter occurs in the M matrix -function matrix_gradient(M_indices::ArrayParamsMap, - M_length::Integer) - rowval = reduce(vcat, M_indices) - colptr = pushfirst!(accumulate((ptr, M_ind) -> ptr + length(M_ind), M_indices, init=1), 1) - return SparseMatrixCSC(M_length, length(M_indices), - colptr, rowval, ones(length(rowval))) -end - -# fill M with parameters -function fill_matrix!(M::AbstractMatrix, M_indices::AbstractArrayParamsMap, - parameters::AbstractVector) - - for (iM, par) in zip(M_indices, parameters) - for index_M in iM - M[index_M] = par - end - end - return M -end - -# range of parameters that are referenced in the matrix -function param_range(mtx_indices::AbstractArrayParamsMap) - - first_i = findfirst(!isempty, mtx_indices) - last_i = findlast(!isempty, mtx_indices) - - if !isnothing(first_i) && !isnothing(last_i) - for i in first_i:last_i - if isempty(mtx_indices[i]) - # TODO show which parameter is missing in which matrix - throw(ErrorException( - "Your parameter vector is not partitioned into directed and undirected effects")) - end - end - end - - return first_i:last_i -end From 0a2bde67d420ae997c66fd48151cd2d8e5f61cb7 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 18:53:33 -0700 Subject: [PATCH 130/174] remove identifier.jl unused, replaced by params() API --- src/StructuralEquationModels.jl | 2 -- src/additional_functions/identifier.jl | 43 -------------------------- test/unit_tests/specification.jl | 14 +++------ 3 files changed, 4 insertions(+), 55 deletions(-) delete mode 100644 src/additional_functions/identifier.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index c3af1cc26..bf093945c 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -58,8 +58,6 @@ include("additional_functions/start_val/start_partable.jl") include("additional_functions/start_val/start_simple.jl") include("additional_functions/artifacts.jl") include("additional_functions/simulation.jl") -# identifier -#include("additional_functions/identifier.jl") # fit measures include("frontend/fit/fitmeasures/AIC.jl") include("frontend/fit/fitmeasures/BIC.jl") diff --git a/src/additional_functions/identifier.jl b/src/additional_functions/identifier.jl deleted file mode 100644 index 2d2bdb694..000000000 --- a/src/additional_functions/identifier.jl +++ /dev/null @@ -1,43 +0,0 @@ -############################################################################################ -# get parameter identifier -############################################################################################ - - -############################################################################################ -# get indices of a Vector of parameter labels -############################################################################################ - -get_identifier_indices(parameters, identifier::Dict{Symbol, Int}) = - [identifier[par] for par in parameters] - -get_identifier_indices(parameters, obj::Union{SemFit, AbstractSemSingle, SemEnsemble, SemImply}) = - get_identifier_indices(parameters, params(obj)) - -function get_identifier_indices(parameters, obj::Union{ParameterTable, RAMMatrices}) - @warn "You are trying to find parameter indices from a ParameterTable or RAMMatrices object. \n - If your model contains user-defined types, this may lead to wrong results. \n - To be on the safe side, try to reference parameters by labels or query the indices from - the constructed model (`get_identifier_indices(parameters, model)`)." maxlog=1 - return get_identifier_indices(parameters, params(obj)) -end - -############################################################################################ -# documentation -############################################################################################ -""" - get_identifier_indices(parameters, model) - -Returns the indices of `parameters`. - -# Arguments -- `parameters::Vector{Symbol}`: parameter labels -- `model`: either a SEM or a fitted SEM - -# Examples -```julia -parameter_indices = get_identifier_indices([:λ₁, λ₂], my_fitted_sem) - -values = solution(my_fitted_sem)[parameter_indices] -``` -""" -function get_identifier_indices end \ No newline at end of file diff --git a/test/unit_tests/specification.jl b/test/unit_tests/specification.jl index 4832d4559..b7e82e84f 100644 --- a/test/unit_tests/specification.jl +++ b/test/unit_tests/specification.jl @@ -3,18 +3,12 @@ @test ram_matrices == RAMMatrices(partable) end -@test get_identifier_indices([:x2, :x10, :x28], model_ml) == [2, 10, 28] - -@testset "get_identifier_indices" begin - pars = [:θ_1, :θ_7, :θ_21] - @test get_identifier_indices(pars, model_ml) == get_identifier_indices(pars, partable) - @test get_identifier_indices(pars, model_ml) == get_identifier_indices(pars, RAMMatrices(partable)) +@testset "params()" begin + @test params(model_ml)[2, 10, 28] == [:x2, :x10, :x28] + @test params(model_ml) == params(partable) + @test params(model_ml) == params(RAMMatrices(partable)) end -# from docstrings: -parameter_indices = get_identifier_indices([:λ₁, λ₂], my_fitted_sem) -values = solution(my_fitted_sem)[parameter_indices] - graph = @StenoGraph begin # measurement model visual → fixed(1.0, 1.0)*x1 + fixed(0.5, 0.5)*x2 + fixed(0.6, 0.8)*x3 From 67873abf9e940b2973ac6c98106f2129aeb29cad Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 20 Mar 2024 19:10:48 -0700 Subject: [PATCH 131/174] rename parameter_type to relation for clarity --- src/frontend/fit/summary.jl | 23 ++++++++------------ src/frontend/specification/ParameterTable.jl | 12 +++++----- src/frontend/specification/RAMMatrices.jl | 20 ++++++++--------- src/frontend/specification/StenoGraphs.jl | 6 ++--- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/frontend/fit/summary.jl b/src/frontend/fit/summary.jl index dcfd819e1..4fd6bb3e1 100644 --- a/src/frontend/fit/summary.jl +++ b/src/frontend/fit/summary.jl @@ -54,13 +54,12 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co sorted_columns = [:to, :estimate, :param, :value_fixed, :start] loading_columns = sort_partially(sorted_columns, columns) header_cols = copy(loading_columns) - replace!(header_cols, :parameter_type => :type) for var in partable.variables[:latent_vars] indicator_indices = findall( (partable.columns[:from] .== var) .& - (partable.columns[:parameter_type] .== :→) .& + (partable.columns[:relation] .== :→) .& (partable.columns[:to] .∈ [partable.variables[:observed_vars]]) ) loading_array = reduce(hcat, check_round(partable.columns[c][indicator_indices]; digits = digits) for c in loading_columns) @@ -76,7 +75,7 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co regression_indices = findall( - (partable.columns[:parameter_type] .== :→) .& + (partable.columns[:relation] .== :→) .& ( ( (partable.columns[:to] .∈ [partable.variables[:observed_vars]]) .& @@ -93,12 +92,11 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co ) ) - sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] regression_columns = sort_partially(sorted_columns, columns) regression_array = reduce(hcat, check_round(partable.columns[c][regression_indices]; digits = digits) for c in regression_columns) regression_columns[2] = Symbol("") - replace!(regression_columns, :parameter_type => :type) print("\n") pretty_table(regression_array; header = regression_columns, tf = PrettyTables.tf_borderless, alignment = :l) @@ -108,16 +106,15 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co variance_indices = findall( - (partable.columns[:parameter_type] .== :↔) .& + (partable.columns[:relation] .== :↔) .& (partable.columns[:to] .== partable.columns[:from]) ) - sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce(hcat, check_round(partable.columns[c][variance_indices]; digits = digits) for c in variance_columns) variance_columns[2] = Symbol("") - replace!(variance_columns, :parameter_type => :type) print("\n") pretty_table(variance_array; header = variance_columns, tf = PrettyTables.tf_borderless, alignment = :l) @@ -127,16 +124,15 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co variance_indices = findall( - (partable.columns[:parameter_type] .== :↔) .& + (partable.columns[:relation] .== :↔) .& (partable.columns[:to] .!= partable.columns[:from]) ) - sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce(hcat, check_round(partable.columns[c][variance_indices]; digits = digits) for c in variance_columns) variance_columns[2] = Symbol("") - replace!(variance_columns, :parameter_type => :type) print("\n") pretty_table(variance_array; header = variance_columns, tf = PrettyTables.tf_borderless, alignment = :l) @@ -144,7 +140,7 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co mean_indices = findall( - (partable.columns[:parameter_type] .== :→) .& + (partable.columns[:relation] .== :→) .& (partable.columns[:from] .== Symbol("1")) ) @@ -152,12 +148,11 @@ function sem_summary(partable::ParameterTable; color = :light_cyan, secondary_co printstyled("Means: \n"; color = color) - sorted_columns = [:from, :parameter_type, :to, :estimate, :param, :value_fixed, :start] + sorted_columns = [:from, :relation, :to, :estimate, :param, :value_fixed, :start] variance_columns = sort_partially(sorted_columns, columns) variance_array = reduce(hcat, check_round(partable.columns[c][mean_indices]; digits = digits) for c in variance_columns) variance_columns[2] = Symbol("") - replace!(variance_columns, :parameter_type => :type) print("\n") pretty_table(variance_array; header = variance_columns, tf = PrettyTables.tf_borderless, alignment = :l) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 7135bcef8..eedcfd248 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -18,7 +18,7 @@ function ParameterTable(; observed_vars::Union{AbstractVector{Symbol}, Nothing}= params::Union{AbstractVector{Symbol}, Nothing}=nothing) columns = ( from = Vector{Symbol}(), - parameter_type = Vector{Symbol}(), + relation = Vector{Symbol}(), to = Vector{Symbol}(), free = Vector{Bool}(), value_fixed = Vector{Float64}(), @@ -116,7 +116,7 @@ end function Base.show(io::IO, partable::ParameterTable) relevant_columns = [ :from, - :parameter_type, + :relation, :to, :free, :value_fixed, @@ -148,7 +148,7 @@ end # Iteration -------------------------------------------------------------------------------- ParameterTableRow = @NamedTuple begin from::Symbol - parameter_type::Symbol + relation::Symbol to::Symbol free::Bool value_fixed::Any @@ -157,7 +157,7 @@ end Base.getindex(partable::ParameterTable, i::Integer) = (from = partable.columns.from[i], - parameter_type = partable.columns.parameter_type[i], + relation = partable.columns.relation[i], to = partable.columns.to[i], free = partable.columns.free[i], value_fixed = partable.columns.value_fixed[i], @@ -198,7 +198,7 @@ function sort_vars!(partable::ParameterTable) # regression edges (excluding intercept) edges = [(from, to) for (reltype, from, to) in - zip(partable.columns.parameter_type, + zip(partable.columns.relation, partable.columns.from, partable.columns.to) if (reltype == :→) && (from != Symbol("1"))] @@ -430,7 +430,7 @@ function lavaan_param_values!(out::AbstractVector, lav_values = partable_lav[:, lav_col] for (from, to, type, id) in zip([view(partable.columns[k], partable_mask) - for k in [:from, :to, :parameter_type, :param]]...) + for k in [:from, :to, :relation, :param]]...) lav_ind = nothing diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 57cf7849b..525f0b656 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -138,31 +138,31 @@ function RAMMatrices(partable::ParameterTable; col_ind = row.from != Symbol("1") ? cols_index[row.from] : nothing if !row.free - if (row.parameter_type == :→) && (row.from == Symbol("1")) + if (row.relation == :→) && (row.from == Symbol("1")) push!(M_consts, row_ind => row.value_fixed) - elseif (row.parameter_type == :→) + elseif (row.relation == :→) push!(A_consts, A_lin_ixs[CartesianIndex(row_ind, col_ind)] => row.value_fixed) - elseif (row.parameter_type == :↔) + elseif (row.relation == :↔) push!(S_consts, S_lin_ixs[CartesianIndex(row_ind, col_ind)] => row.value_fixed) if row_ind != col_ind # symmetric push!(S_consts, S_lin_ixs[CartesianIndex(col_ind, row_ind)] => row.value_fixed) end else - error("Unsupported parameter type: $(row.parameter_type)") + error("Unsupported relation: $(row.relation)") end else par_ind = params_index[row.param] - if (row.parameter_type == :→) && (row.from == Symbol("1")) + if (row.relation == :→) && (row.from == Symbol("1")) push!(M_inds[par_ind], row_ind) - elseif row.parameter_type == :→ + elseif row.relation == :→ push!(A_inds[par_ind], A_lin_ixs[CartesianIndex(row_ind, col_ind)]) - elseif row.parameter_type == :↔ + elseif row.relation == :↔ push!(S_inds[par_ind], S_lin_ixs[CartesianIndex(row_ind, col_ind)]) if row_ind != col_ind # symmetric push!(S_inds[par_ind], S_lin_ixs[CartesianIndex(col_ind, row_ind)]) end else - error("Unsupported parameter type: $(row.parameter_type)") + error("Unsupported relation: $(row.relation)") end end end @@ -252,7 +252,7 @@ end ### Additional Functions ############################################################################################ -function matrix_to_parameter_type(matrix::Symbol) +function matrix_to_relation(matrix::Symbol) if matrix == :A return :→ elseif matrix == :S @@ -279,7 +279,7 @@ function partable_row(val, index, matrix::Symbol, return ( from = from, - parameter_type = matrix_to_parameter_type(matrix), + relation = matrix_to_relation(matrix), to = to, free = free, value_fixed = free ? 0.0 : val, diff --git a/src/frontend/specification/StenoGraphs.jl b/src/frontend/specification/StenoGraphs.jl index 5bdf9ce93..57fcb1134 100644 --- a/src/frontend/specification/StenoGraphs.jl +++ b/src/frontend/specification/StenoGraphs.jl @@ -42,7 +42,7 @@ function ParameterTable(graph::AbstractStenoGraph; observed_vars = observed_vars, params = params) from = resize!(partable.columns.from, n) - parameter_type = resize!(partable.columns.parameter_type, n) + relation = resize!(partable.columns.relation, n) to = resize!(partable.columns.to, n) free = fill!(resize!(partable.columns.free, n), true) value_fixed = fill!(resize!(partable.columns.value_fixed, n), NaN) @@ -56,9 +56,9 @@ function ParameterTable(graph::AbstractStenoGraph; from[i] = edge.src.node to[i] = edge.dst.node if edge isa DirectedEdge - parameter_type[i] = :→ + relation[i] = :→ elseif edge isa UndirectedEdge - parameter_type[i] = :↔ + relation[i] = :↔ else throw(ArgumentError("The graph contains an unsupported edge of type $(typeof(edge)).")) end From 3f53a7479229c75acf90d192a37de188b52547b3 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:02:40 -0700 Subject: [PATCH 132/174] materialize!(Symm/LowTri/UpTri) --- src/additional_functions/params_array.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index d71e172c9..efe926af6 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -115,6 +115,11 @@ materialize(::Type{T}, arr::ParamsArray, param_values::AbstractVector) where T = materialize(arr::ParamsArray, param_values::AbstractVector{T}) where T = materialize(Union{T, eltype(arr)}, arr, param_values) +# the hack to update the structured matrix (should be fine since the structure is imposed by ParamsMatrix) +materialize!(dest::Union{Symmetric,LowerTriangular,UpperTriangular}, src::ParamsMatrix{<:Any}, + param_values::AbstractVector; kwargs...) = + materialize!(parent(dest), src, param_values; kwargs...) + function sparse_materialize(::Type{T}, arr::ParamsMatrix, param_values::AbstractVector) where T From 0b27e8cb801537f3147552cf33fa5e1b19999b8e Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:10:43 -0700 Subject: [PATCH 133/174] generic imply: keep F sparse --- src/imply/RAM/generic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index e20fed166..dc5e9a54b 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -109,7 +109,7 @@ function RAM(; nan_params = fill(NaN, n_par) A_pre = materialize(ram_matrices.A, nan_params) S_pre = materialize(ram_matrices.S, nan_params) - F = Matrix(ram_matrices.F) + F = copy(ram_matrices.F) A_pre = check_acyclic(A_pre, ram_matrices.A) From af277c7d9f1b7b5203d3e8ef33de473539471b40 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:12:14 -0700 Subject: [PATCH 134/174] generic imply: impose matrix constraints define S, A, and I_A fields as symmetric or low/up triangular --- src/imply/RAM/generic.jl | 46 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index dc5e9a54b..3f4e518ff 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -106,18 +106,19 @@ function RAM(; n_var = nvars(ram_matrices) #preallocate arrays - nan_params = fill(NaN, n_par) - A_pre = materialize(ram_matrices.A, nan_params) - S_pre = materialize(ram_matrices.S, nan_params) + rand_params = randn(Float64, n_par) + A_pre = check_acyclic(materialize(ram_matrices.A, rand_params)) + S_pre = Symmetric(materialize(ram_matrices.S, rand_params)) F = copy(ram_matrices.F) - A_pre = check_acyclic(A_pre, ram_matrices.A) - # pre-allocate some matrices - Σ = zeros(n_obs, n_obs) + Σ = Symmetric(zeros(n_obs, n_obs)) F⨉I_A⁻¹ = zeros(n_obs, n_var) F⨉I_A⁻¹S = zeros(n_obs, n_var) - I_A = similar(A_pre) + I_A = convert(Matrix, I - A_pre) + I_A = istril(I_A) ? LowerTriangular(I_A) : + istriu(I_A) ? UpperTriangular(I_A) : + I_A if gradient_required ∇A = sparse_gradient(ram_matrices.A) @@ -130,7 +131,7 @@ function RAM(; # μ if meanstructure MS = HasMeanStructure - M_pre = materialize(ram_matrices.M, nan_params) + M_pre = materialize(ram_matrices.M, rand_params) ∇M = gradient_required ? sparse_gradient(ram_matrices.M) : nothing μ = zeros(n_obs) else @@ -153,7 +154,7 @@ function RAM(; F⨉I_A⁻¹, F⨉I_A⁻¹S, I_A, - copy(I_A), + similar(parent(I_A)), ∇A, ∇S, @@ -172,7 +173,7 @@ function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingl materialize!(imply.M, imply.ram_matrices.M, params) end - @inbounds for (j, I_Aj, Aj) in zip(axes(imply.A, 2), eachcol(imply.I_A), eachcol(imply.A)) + @inbounds for (j, I_Aj, Aj) in zip(axes(imply.A, 2), eachcol(parent(imply.I_A)), eachcol(imply.A)) for i in axes(imply.A, 1) I_Aj[i] = ifelse(i == j, 1, 0) - Aj[i] end @@ -187,7 +188,7 @@ function update!(targets::EvaluationTargets, imply::RAM, model::AbstractSemSingl end mul!(imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹, imply.S) - mul!(imply.Σ, imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹') + mul!(parent(imply.Σ), imply.F⨉I_A⁻¹S, imply.F⨉I_A⁻¹') if MeanStructure(imply) === HasMeanStructure mul!(imply.μ, imply.F⨉I_A⁻¹, imply.M) @@ -213,22 +214,21 @@ end ### additional functions ############################################################################################ -function check_acyclic(A_pre::AbstractMatrix, A::ParamsMatrix) - # fill copy of A with random parameters - A_rand = materialize(A, rand(nparams(A))) - +function check_acyclic(A::AbstractMatrix) # check if the model is acyclic - acyclic = isone(det(I-A_rand)) + acyclic = isone(det(I-A)) # check if A is lower or upper triangular - if istril(A_rand) + if istril(A) @info "A matrix is lower triangular" - return LowerTriangular(A_pre) - elseif istriu(A_rand) + return LowerTriangular(A) + elseif istriu(A) @info "A matrix is upper triangular" - return UpperTriangular(A_pre) - elseif acyclic - @info "Your model is acyclic, specifying the A Matrix as either Upper or Lower Triangular can have great performance benefits.\n" maxlog=1 - return A_pre + return UpperTriangular(A) + else + if acyclic + @info "Your model is acyclic, specifying the A Matrix as either Upper or Lower Triangular can have great performance benefits.\n" maxlog=1 + end + return A end end \ No newline at end of file From d6d5849d1e2d90d16558f228f80ecf024503e474 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 15:13:29 -0700 Subject: [PATCH 135/174] neumann_series(): avoid endless loop if the series don't converge --- src/additional_functions/helper.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index dacbd9e23..9335e17b6 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -1,10 +1,14 @@ -function neumann_series(mat::SparseMatrixCSC) +# Neumann seriess representation of (I - mat)⁻¹ +function neumann_series(mat::SparseMatrixCSC; maxn::Integer = size(mat, 1)) inverse = I + mat next_term = mat^2 + n = 1 while nnz(next_term) != 0 + (n <= maxn) || error("Neumann series did not converge in $maxn steps") inverse += next_term next_term *= mat + n += 1 end return inverse From a22247bc5477e68ac92908f4efb9ef6bfba507bf Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 16:13:34 -0700 Subject: [PATCH 136/174] ParamsArray: faster sparse materialize! --- src/additional_functions/params_array.jl | 81 +++++++++++++++++------ src/frontend/specification/RAMMatrices.jl | 2 +- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/src/additional_functions/params_array.jl b/src/additional_functions/params_array.jl index efe926af6..c20528b3b 100644 --- a/src/additional_functions/params_array.jl +++ b/src/additional_functions/params_array.jl @@ -2,10 +2,14 @@ Array with partially parameterized elements. """ struct ParamsArray{T, N} <: AbstractArray{T, N} - linear_indices::Vector{Int} - param_ptr::Vector{Int} - constants::Vector{Pair{Int, T}} - size::NTuple{N, Int} + linear_indices::Vector{Int} # linear indices of the parameter refs in the destination array + nz_indices::Vector{Int} # indices of the parameters refs in nonzero elements vector + # (including the constants) ordered by the linear index + param_ptr::Vector{Int} # i-th element marks the start of the range in linear/nonzero + # indices arrays that corresponds to the i-th parameter + # (nparams + 1 elements) + constants::Vector{Tuple{Int, Int, T}} # linear index, index in nonzero vector, value + size::NTuple{N, Int} # size of the destination array end ParamsVector{T} = ParamsArray{T, 1} @@ -16,8 +20,16 @@ function ParamsArray{T,N}(params_map::AbstractVector{<:AbstractVector{Int}}, size::NTuple{N, Int}) where {T,N} params_ptr = pushfirst!(accumulate((ptr, inds) -> ptr + length(inds), params_map, init=1), 1) - return ParamsArray{T,N}(reduce(vcat, params_map, init=Vector{Int}()), params_ptr, - constants, size) + param_lin_inds = reduce(vcat, params_map, init=Vector{Int}()) + nz_lin_inds = unique!(sort!([param_lin_inds; first.(constants)])) + if length(nz_lin_inds) < length(param_lin_inds) + length(constants) + throw(ArgumentError("Duplicate linear indices in the parameterized array")) + end + return ParamsArray{T,N}(param_lin_inds, + searchsortedfirst.(Ref(nz_lin_inds), param_lin_inds), + params_ptr, + [(c[1], searchsortedfirst(nz_lin_inds, c[1]), c[2]) + for c in constants], size) end function ParamsArray{T,N}(arr::AbstractArray{<:Any, N}, params::AbstractVector{Symbol}; @@ -46,6 +58,7 @@ ParamsArray{T}(arr::AbstractArray{<:Any, N}, params::AbstractVector{Symbol}; kwa ParamsArray{T,N}(arr, params; kwargs...) nparams(arr::ParamsArray) = length(arr.param_ptr) - 1 +SparseArrays.nnz(arr::ParamsArray) = length(arr.linear_indices) + length(arr.constants) Base.size(arr::ParamsArray) = arr.size Base.size(arr::ParamsArray, i::Integer) = arr.size[i] @@ -90,7 +103,7 @@ function materialize!(dest::AbstractArray{<:Any, N}, src::ParamsArray{<:Any, N}, Z = eltype(dest) <: Number ? eltype(dest) : eltype(src) set_zeros && fill!(dest, zero(Z)) if set_constants - @inbounds for (i, val) in src.constants + @inbounds for (i, _, val) in src.constants dest[i] = val end end @@ -102,6 +115,31 @@ function materialize!(dest::AbstractArray{<:Any, N}, src::ParamsArray{<:Any, N}, return dest end +function materialize!(dest::SparseMatrixCSC, src::ParamsMatrix, + param_values::AbstractVector; + set_constants::Bool = true, + set_zeros::Bool = false) + set_zeros && throw(ArgumentError("Cannot set zeros for sparse matrix")) + size(dest) == size(src) || + throw(DimensionMismatch("Parameters ($(size(params_arr))) and destination ($(size(dest))) array sizes don't match")) + nparams(src) == length(param_values) || + throw(DimensionMismatch("Number of values ($(length(param_values))) does not match the number of parameters ($(nparams(src)))")) + + nnz(dest) == nnz(src) || + throw(DimensionMismatch("Number of non-zero elements ($(nnz(dest))) does not match the number of parameter references and constants ($(nnz(src)))")) + if set_constants + @inbounds for (_, j, val) in src.constants + dest.nzval[j] = val + end + end + @inbounds for (i, val) in enumerate(param_values) + for j in param_occurences_range(src, i) + dest.nzval[src.nz_indices[j]] = val + end + end + return dest +end + """ materialize([T], src::ParamsArray{<:Any, N}, param_values::AbstractVector{T}) where T @@ -124,24 +162,27 @@ function sparse_materialize(::Type{T}, arr::ParamsMatrix, param_values::AbstractVector) where T nparams(arr) == length(param_values) || - throw(DimensionMismatch("Number of values ($(length(param)values))) does not match the number of parameter ($(nparams(arr)))")) - # constant values in sparse matrix - cvals = [T(v) for (_, v) in arr.constants] - # parameter values in sparse matrix - parvals = Vector{T}(undef, length(arr.linear_indices)) + throw(DimensionMismatch("Number of values ($(length(param_values))) does not match the number of parameter ($(nparams(arr)))")) + + nz_vals = Vector{T}(undef, nnz(arr)) + nz_lininds = Vector{Int}(undef, nnz(arr)) + # fill constants + @inbounds for (lin_ind, nz_ind, val) in arr.constants + nz_vals[nz_ind] = val + nz_lininds[nz_ind] = lin_ind + end + # fill parameters @inbounds for (i, val) in enumerate(param_values) for j in param_occurences_range(arr, i) - parvals[j] = val + nz_ind = arr.nz_indices[j] + nz_vals[nz_ind] = val + nz_lininds[nz_ind] = arr.linear_indices[j] end end - nzixs = [first.(arr.constants); arr.linear_indices] - ixorder = sortperm(nzixs) - nzixs = nzixs[ixorder] - nzvals = [cvals; parvals][ixorder] arr_ixs = CartesianIndices(size(arr)) - return sparse([arr_ixs[i][1] for i in nzixs], - [arr_ixs[i][2] for i in nzixs], - nzvals, size(arr)...) + return sparse([arr_ixs[i][1] for i in nz_lininds], + [arr_ixs[i][2] for i in nz_lininds], + nz_vals, size(arr)...) end sparse_materialize(arr::ParamsArray, params::AbstractVector{T}) where T = diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 525f0b656..b842708b6 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -314,7 +314,7 @@ function append_rows!(partable::ParameterTable, end # add constants - for (i, val) in arr.constants + for (i, _, val) in arr.constants arr_ix = arr_ixs[i] skip_symmetric && (arr_ix ∈ visited_indices) && continue push!(partable, partable_row(val, arr_ix, arr_name, position_names, free=false)) From 25c64326ccdf05453d39f2fbd5c5ebab8916432a Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 18:33:03 -0700 Subject: [PATCH 137/174] RAM: reuse sigma array --- src/loss/ML/ML.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 1af993246..26025c4db 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -178,7 +178,12 @@ function evaluate!( ∇A = implied.∇A ∇S = implied.∇S - C = F⨉I_A⁻¹'*(I-Σ⁻¹Σₒ)*Σ⁻¹*F⨉I_A⁻¹ + # reuse Σ⁻¹Σₒ to calculate I-Σ⁻¹Σₒ + one_Σ⁻¹Σₒ = Σ⁻¹Σₒ + one_Σ⁻¹Σₒ.*= -1 + one_Σ⁻¹Σₒ[diagind(one_Σ⁻¹Σₒ)] .+= 1 + + C = F⨉I_A⁻¹'*one_Σ⁻¹Σₒ*Σ⁻¹*F⨉I_A⁻¹ mul!(gradient, ∇A', vec(C*S*I_A⁻¹'), 2, 0) mul!(gradient, ∇S', vec(C), 1, 1) From 28a9baff81dad5254102d24adb86a453ffb79226 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:01:45 -0700 Subject: [PATCH 138/174] RAM: optional sparse Sigma matrix --- src/imply/RAM/generic.jl | 3 ++- src/loss/ML/ML.jl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index 3f4e518ff..aff6a6c54 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -96,6 +96,7 @@ function RAM(; #vech = false, gradient_required = true, meanstructure = false, + sparse_S::Bool = true, kwargs...) ram_matrices = convert(RAMMatrices, specification) @@ -108,7 +109,7 @@ function RAM(; #preallocate arrays rand_params = randn(Float64, n_par) A_pre = check_acyclic(materialize(ram_matrices.A, rand_params)) - S_pre = Symmetric(materialize(ram_matrices.S, rand_params)) + S_pre = Symmetric((sparse_S ? sparse_materialize : materialize)(ram_matrices.S, rand_params)) F = copy(ram_matrices.F) # pre-allocate some matrices diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 26025c4db..087c0ac63 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -184,7 +184,7 @@ function evaluate!( one_Σ⁻¹Σₒ[diagind(one_Σ⁻¹Σₒ)] .+= 1 C = F⨉I_A⁻¹'*one_Σ⁻¹Σₒ*Σ⁻¹*F⨉I_A⁻¹ - mul!(gradient, ∇A', vec(C*S*I_A⁻¹'), 2, 0) + mul!(gradient, ∇A', vec(C*mul!(similar(C), S, I_A⁻¹')), 2, 0) mul!(gradient, ∇S', vec(C), 1, 1) if MeanStructure(implied) === HasMeanStructure From 1d6ab6fc6b096bf26d16ddd6eb4232c1e005f181 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Fri, 22 Mar 2024 18:34:13 -0700 Subject: [PATCH 139/174] RAM: declare (I-A)^-1 up/low tri too --- src/imply/RAM/generic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imply/RAM/generic.jl b/src/imply/RAM/generic.jl index aff6a6c54..e53768c20 100644 --- a/src/imply/RAM/generic.jl +++ b/src/imply/RAM/generic.jl @@ -155,7 +155,7 @@ function RAM(; F⨉I_A⁻¹, F⨉I_A⁻¹S, I_A, - similar(parent(I_A)), + similar(I_A), ∇A, ∇S, From 5551edcd3906d70b2ba70d3c2b989e9a0bf36f81 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 16:05:53 -0700 Subject: [PATCH 140/174] cleanup update_partable!() --- .../specification/EnsembleParameterTable.jl | 19 ++++-- src/frontend/specification/ParameterTable.jl | 65 +++++++++++-------- 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/frontend/specification/EnsembleParameterTable.jl b/src/frontend/specification/EnsembleParameterTable.jl index bfc95264a..fe4a2f5c8 100644 --- a/src/frontend/specification/EnsembleParameterTable.jl +++ b/src/frontend/specification/EnsembleParameterTable.jl @@ -119,12 +119,23 @@ Base.getindex(partable::EnsembleParameterTable, group) = partable.tables[group] ### Update Partable from Fitted Model ############################################################################################ -# update generic --------------------------------------------------------------------------- +# TODO group-specific values (via dictionary of parameter values?) + function update_partable!(partables::EnsembleParameterTable, + column::Symbol, + param_values::AbstractDict{Symbol}, + default::Any = nothing) + for partable in values(partables.tables) + update_partable!(partable, column, param_values, column, default) + end + return partables +end + +function update_partable!(partables::EnsembleParameterTable, column::Symbol, params::AbstractVector, param_values::AbstractVector, - column::Symbol) + default::Any = nothing) for partable in values(partables.tables) - update_partable!(partable, params, param_values, column) + update_partable!(partable, column, params, param_values, default) end return partables -end \ No newline at end of file +end diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index eedcfd248..fbfac1fc4 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -259,36 +259,43 @@ end ############################################################################################ # update generic --------------------------------------------------------------------------- - function update_partable!(partable::ParameterTable, - params::AbstractVector{Symbol}, - values::AbstractVector, - column::Symbol) - length(params) == length(values) || - throw(ArgumentError("The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same")) + column::Symbol, + param_values::AbstractDict{Symbol}, + default::Any = nothing) coldata = partable.columns[column] - fixed_values = partable.columns.value_fixed - param_index = Dict(zip(params, eachindex(params))) resize!(coldata, length(partable)) for (i, id) in enumerate(partable.columns.param) - coldata[i] = id != :const ? - values[param_index[id]] : - fixed_values[i] + (id == :const) && continue + if haskey(param_values, id) + coldata[i] = param_values[id] + else + if isnothing(default) + throw(KeyError(id)) + elseif (default isa AbstractVector) && + (length(default) == length(partable)) + coldata[i] = default[i] + else + coldata[i] = default + end + end end return partable end -""" - update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, values, column) - -Write `vec` to `column` of `partable`. - -# Arguments -- `vec::Vector`: has to be in the same order as the `model` parameters -""" -update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, - values::AbstractVector, column::Symbol) = - update_partable!(partable, params(sem_fit), values, column) +function update_partable!(partable::ParameterTable, + column::Symbol, + params::AbstractVector{Symbol}, + values::AbstractVector, + default::Any = nothing) + length(params) == length(values) || + throw(ArgumentError("The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same")) + params_dict = Dict(zip(params, values)) + if length(params_dict) != length(params) + throw(ArgumentError("Duplicate parameter names in `params`")) + end + update_partable!(partable, column, params_dict, default) +end # update estimates ------------------------------------------------------------------------- """ @@ -298,8 +305,13 @@ update_partable!(partable::AbstractParameterTable, sem_fit::SemFit, Write parameter estimates from `sem_fit` to the `:estimate` column of `partable` """ +update_estimate!(partable::ParameterTable, sem_fit::SemFit) = + update_partable!(partable, :estimate, params(sem_fit), sem_fit.solution, + partable.columns.value_fixed) + +# fallback method for ensemble update_estimate!(partable::AbstractParameterTable, sem_fit::SemFit) = - update_partable!(partable, sem_fit, sem_fit.solution, :estimate) + update_partable!(partable, :estimate, params(sem_fit), sem_fit.solution) # update starting values ------------------------------------------------------------------- """ @@ -314,7 +326,8 @@ Write starting values from `sem_fit` or `start_val` to the `:estimate` column of - `kwargs...`: are passed to `start_val` """ update_start!(partable::AbstractParameterTable, sem_fit::SemFit) = - update_partable!(partable, sem_fit, sem_fit.start_val, :start) + update_partable!(partable, :start, params(sem_fit), sem_fit.start_val, + partable.columns.value_fixed) function update_start!( partable::AbstractParameterTable, @@ -324,7 +337,7 @@ function update_start!( if !(start_val isa Vector) start_val = start_val(model; kwargs...) end - return update_partable!(partable, params(model), start_val, :start) + return update_partable!(partable, :start, params(model), start_val) end # update partable standard errors ---------------------------------------------------------- @@ -347,7 +360,7 @@ function update_se_hessian!( fit::SemFit; method = :finitediff) se = se_hessian(fit; method = method) - return update_partable!(partable, fit, se, :se) + return update_partable!(partable, :se, params(fit), se) end """ From 73cedb8858731cd904074aa7490cf4759c1591c0 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 21:28:43 -0700 Subject: [PATCH 141/174] cleanup start_vals handling * unify processing of starting values by all optimizers * support dictionaries of values * remove start_val() function (logic moved to prepare_start_params()) --- src/StructuralEquationModels.jl | 1 - .../start_val/start_partable.jl | 42 ++++++------------- .../start_val/start_val.jl | 32 -------------- src/optimizer/NLopt.jl | 24 +++++------ src/optimizer/documentation.jl | 35 +++++++++++++--- src/optimizer/optim.jl | 16 +++---- 6 files changed, 59 insertions(+), 91 deletions(-) delete mode 100644 src/additional_functions/start_val/start_val.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index bf093945c..ca088094b 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -52,7 +52,6 @@ include("optimizer/optim.jl") include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") -include("additional_functions/start_val/start_val.jl") include("additional_functions/start_val/start_fabin3.jl") include("additional_functions/start_val/start_partable.jl") include("additional_functions/start_val/start_simple.jl") diff --git a/src/additional_functions/start_val/start_partable.jl b/src/additional_functions/start_val/start_partable.jl index 5f90ae3c2..930d5dec1 100644 --- a/src/additional_functions/start_val/start_partable.jl +++ b/src/additional_functions/start_val/start_partable.jl @@ -1,42 +1,26 @@ """ start_parameter_table(model; parameter_table) - + Return a vector of starting values taken from `parameter_table`. """ -function start_parameter_table end - -# splice model and loss functions -function start_parameter_table(model::AbstractSemSingle; kwargs...) - return start_parameter_table( - model.observed, - model.imply, - model.optimizer, - model.loss.functions...; - kwargs...) -end +start_parameter_table(model::AbstractSemSingle; + partable::ParameterTable, kwargs...) = + start_parameter_table(partable) -# RAM(Symbolic) -function start_parameter_table(observed, imply, optimizer, args...; kwargs...) - return start_parameter_table( - ram_matrices(imply); - kwargs...) -end +function start_parameter_table(partable::ParameterTable) + start_vals = zeros(eltype(partable.columns.start), nparams(partable)) + param_indices = Dict(param => i for (i, param) in enumerate(params(partable))) -function start_parameter_table(ram::RAMMatrices; partable::ParameterTable, kwargs...) - - start_val = zeros(0) - - param_indices = Dict(param => i for (i, param) in enumerate(params(ram))) - - for (i, param) in enumerate(partable.columns.param) + for (param, startval) in zip(partable.columns.param, + partable.columns.start) + (param == :const) && continue par_ind = get(param_indices, param, nothing) if !isnothing(par_ind) - isfinite(partable.start[i]) && (start_val[i] = partable.start[i]) + isfinite(startval) && (start_vals[par_ind] = startval) else - throw(ErrorException("Parameter $(param) is not in the parameter table.")) + throw(ErrorException("Parameter $(param) not found in the model.")) end end - return start_val - + return start_vals end \ No newline at end of file diff --git a/src/additional_functions/start_val/start_val.jl b/src/additional_functions/start_val/start_val.jl deleted file mode 100644 index 120212c49..000000000 --- a/src/additional_functions/start_val/start_val.jl +++ /dev/null @@ -1,32 +0,0 @@ -""" - start_val(model) - -Return a vector of starting values. -Defaults are FABIN 3 starting values for single models and simple starting values for -ensemble models. -""" -function start_val end -# Single Models ---------------------------------------------------------------------------- - -# splice model and loss functions -start_val(model::AbstractSemSingle; kwargs...) = - start_val( - model, - model.observed, - model.imply, - model.optimizer, - model.loss.functions...; - kwargs...) - -# Fabin 3 starting values for RAM(Symbolic) -start_val( - model, - observed, - imply, - optimizer, - args...; - kwargs...) = - start_fabin3(model; kwargs...) - -# Ensemble Models -------------------------------------------------------------------------- -start_val(model::SemEnsemble; kwargs...) = start_simple(model; kwargs...) \ No newline at end of file diff --git a/src/optimizer/NLopt.jl b/src/optimizer/NLopt.jl index dee309d46..2dc9a070f 100644 --- a/src/optimizer/NLopt.jl +++ b/src/optimizer/NLopt.jl @@ -25,34 +25,30 @@ end # sem_fit method function sem_fit( optimizer::SemOptimizerNLopt, - model::AbstractSem; - start_val = start_val, kwargs...) - - # starting values - if !isa(start_val, AbstractVector) - start_val = start_val(model; kwargs...) - end + model::AbstractSem, + start_params::AbstractVector; + kwargs...) # construct the NLopt problem opt = construct_NLopt_problem( - model.optimizer.algorithm, - model.optimizer.options, - length(start_val)) - set_NLopt_constraints!(opt, model.optimizer) + model.optimizer.algorithm, + model.optimizer.options, + length(start_params)) + set_NLopt_constraints!(opt, model.optimizer) opt.min_objective = (par, G) -> evaluate!(eltype(par), !isnothing(G) && !isempty(G) ? G : nothing, nothing, model, par) if !isnothing(model.optimizer.local_algorithm) opt_local = construct_NLopt_problem( model.optimizer.local_algorithm, model.optimizer.local_options, - length(start_val)) + length(start_params)) opt.local_optimizer = opt_local end # fit - result = NLopt.optimize(opt, start_val) + result = NLopt.optimize(opt, start_params) - return SemFit_NLopt(result, model, start_val, opt) + return SemFit_NLopt(result, model, start_params, opt) end ############################################################################################ diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index 7c17e6ce2..34b78c7f9 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -5,16 +5,17 @@ Return the fitted `model`. # Arguments - `model`: `AbstractSem` to fit -- `start_val`: vector of starting values or function to compute starting values (1) +- `start_val`: a vector or a dictionary of starting parameter values, + or function to compute them (1) - `kwargs...`: keyword arguments, passed to starting value functions -(1) available options are `start_fabin3`, `start_simple` and `start_partable`. +(1) available functions are `start_fabin3`, `start_simple` and `start_partable`. For more information, we refer to the individual documentations and the online documentation on [Starting values](@ref). # Examples ```julia sem_fit( - my_model; + my_model; start_val = start_simple, start_covariances_latent = 0.5) ``` @@ -22,8 +23,32 @@ sem_fit( function sem_fit end # dispatch on optimizer -sem_fit(model::AbstractSem; kwargs...) = sem_fit(model.optimizer, model; kwargs...) +function sem_fit(model::AbstractSem; start_val = nothing, kwargs...) + start_params = prepare_start_params(start_val, model; kwargs...) + sem_fit(model.optimizer, model, start_params; kwargs...) +end # fallback method -sem_fit(optimizer::SemOptimizer, model::AbstractSem; kwargs...) = +sem_fit(optimizer::SemOptimizer, model::AbstractSem, start_params; kwargs...) = error("Optimizer $(optimizer) support not implemented.") + +function prepare_start_params(start_val, model::AbstractSem; kwargs...) + if isnothing(start_val) + # default function for starting parameters + # FABIN3 for single models, simple algorithm for ensembles + start_val = model isa AbstractSemSingle ? + start_fabin3(model; kwargs...) : + start_simple(model; kwargs...) + end + if start_val isa AbstractVector + (length(start_val) == nparams(model)) || + throw(DimensionMismatch("The length of `start_val` vector ($(length(start_val))) does not match the number of model parameters ($(nparams(model))).")) + elseif start_val isa AbstractDict + start_val = [start_val[param] for param in params(model)] # convert to a vector + else # function + start_val = start_val(model; kwargs...) + end + @assert start_val isa AbstractVector + @assert length(start_val) == nparams(model) + return start_val +end diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index f855836c6..bbb0137d5 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -1,8 +1,8 @@ ## connect to Optim.jl as backend function SemFit( - optimization_result::Optim.MultivariateOptimizationResults, - model::AbstractSem, + optimization_result::Optim.MultivariateOptimizationResults, + model::AbstractSem, start_val) return SemFit( optimization_result.minimum, @@ -19,19 +19,15 @@ convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res) function sem_fit( optim::SemOptimizerOptim, - model::AbstractSem; - start_val = start_val, + model::AbstractSem, + start_params::AbstractVector; kwargs...) - if !isa(start_val, AbstractVector) - start_val = start_val(model; kwargs...) - end - result = Optim.optimize( Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), - start_val, + start_params, model.optimizer.algorithm, model.optimizer.options) - return SemFit(result, model, start_val) + return SemFit(result, model, start_params) end From d7ae28fdf6d99991471e4e907404012c6579c5ab Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sat, 23 Mar 2024 16:12:53 -0700 Subject: [PATCH 142/174] ML: refactor to minimize allocs * preallocate matrices for intermediate gradient calculation * call mul!() with these preallocating matrices * annotate mul!() arguments as triangular/symmetric to use faster routines --- src/loss/ML/ML.jl | 65 ++++++++++--------- test/examples/political_democracy/by_parts.jl | 4 +- .../recover_parameters_twofact.jl | 2 +- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/src/loss/ML/ML.jl b/src/loss/ML/ML.jl index 087c0ac63..a9dbe999d 100644 --- a/src/loss/ML/ML.jl +++ b/src/loss/ML/ML.jl @@ -27,29 +27,35 @@ Analytic gradients are available, and for models without a meanstructure, also a ## Implementation Subtype of `SemLossFunction`. """ -struct SemML{HE<:HessianEvaluation,INV,M,M2} <: SemLossFunction{HE} - Σ⁻¹::INV - Σ⁻¹Σₒ::M - meandiff::M2 +struct SemML{HE<:HessianEvaluation,M} <: SemLossFunction{HE} + # pre-allocated arrays to store intermediate results in evaluate!() + obsXobs_1::M + obsXobs_2::M + obsXobs_3::M + obsXvar_1::M + varXvar_1::M + varXvar_2::M + varXvar_3::M end ############################################################################################ ### Constructors ############################################################################################ -SemML{HE}(args...) where {HE <: HessianEvaluation} = - SemML{HE, map(typeof, args)...}(args...) - function SemML(; observed::SemObserved, + specification::SemSpecification, approximate_hessian::Bool = false, kwargs...) - obsmean = obs_mean(observed) - obscov = obs_cov(observed) - meandiff = isnothing(obsmean) ? nothing : copy(obsmean) - - return SemML{approximate_hessian ? ApproximateHessian : ExactHessian}( - similar(parent(obscov)), similar(parent(obscov)), - meandiff) + obsXobs = parent(obs_cov(observed)) + nobs = nobserved_vars(specification) + nvar = nvars(specification) + + return SemML{approximate_hessian ? ApproximateHessian : ExactHessian, + typeof(obsXobs)}( + similar(obsXobs), similar(obsXobs), similar(obsXobs), + similar(obsXobs, (nobs, nvar)), + similar(obsXobs, (nvar, nvar)), similar(obsXobs, (nvar, nvar)), + similar(obsXobs, (nvar, nvar))) end ############################################################################################ @@ -73,10 +79,8 @@ function evaluate!( Σ = implied.Σ Σₒ = obs_cov(observed(model)) - Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ - Σ⁻¹ = semml.Σ⁻¹ - copyto!(Σ⁻¹, Σ) + Σ⁻¹ = copy!(semml.obsXobs_1, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) if !isposdef(Σ_chol) #@warn "∑⁻¹ is not positive definite" @@ -87,7 +91,7 @@ function evaluate!( end ld = logdet(Σ_chol) Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + Σ⁻¹Σₒ = mul!(semml.obsXobs_2, Σ⁻¹, Σₒ) isnothing(objective) || (objective = ld + tr(Σ⁻¹Σₒ)) if MeanStructure(implied) === HasMeanStructure @@ -100,12 +104,12 @@ function evaluate!( ∇Σ = implied.∇Σ ∇μ = implied.∇μ μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ - mul!(gradient, ∇Σ', vec(Σ⁻¹*(I - Σₒ*Σ⁻¹ - μ₋*μ₋ᵀΣ⁻¹))) + mul!(gradient, ∇Σ', vec(Σ⁻¹*(I - mul!(semml.obsXobs_3, Σₒ, Σ⁻¹) - μ₋*μ₋ᵀΣ⁻¹))) mul!(gradient, ∇μ', μ₋ᵀΣ⁻¹', -2, 1) end elseif !isnothing(gradient) || !isnothing(hessian) ∇Σ = implied.∇Σ - Σ⁻¹ΣₒΣ⁻¹ = Σ⁻¹Σₒ*Σ⁻¹ + Σ⁻¹ΣₒΣ⁻¹ = mul!(semml.obsXobs_3, Σ⁻¹Σₒ, Σ⁻¹) J = vec(Σ⁻¹ - Σ⁻¹ΣₒΣ⁻¹)' if !isnothing(gradient) mul!(gradient, ∇Σ', J') @@ -144,10 +148,8 @@ function evaluate!( Σ = implied.Σ Σₒ = obs_cov(observed(model)) - Σ⁻¹Σₒ = semml.Σ⁻¹Σₒ - Σ⁻¹ = semml.Σ⁻¹ - copyto!(Σ⁻¹, Σ) + Σ⁻¹ = copy!(semml.obsXobs_1, Σ) Σ_chol = cholesky!(Symmetric(Σ⁻¹); check = false) if !isposdef(Σ_chol) #@warn "Σ⁻¹ is not positive definite" @@ -158,7 +160,7 @@ function evaluate!( end ld = logdet(Σ_chol) Σ⁻¹ = LinearAlgebra.inv!(Σ_chol) - mul!(Σ⁻¹Σₒ, Σ⁻¹, Σₒ) + Σ⁻¹Σₒ = mul!(semml.obsXobs_2, Σ⁻¹, Σₒ) if !isnothing(objective) objective = ld + tr(Σ⁻¹Σₒ) @@ -180,11 +182,16 @@ function evaluate!( # reuse Σ⁻¹Σₒ to calculate I-Σ⁻¹Σₒ one_Σ⁻¹Σₒ = Σ⁻¹Σₒ - one_Σ⁻¹Σₒ.*= -1 + lmul!(-1, one_Σ⁻¹Σₒ) one_Σ⁻¹Σₒ[diagind(one_Σ⁻¹Σₒ)] .+= 1 - C = F⨉I_A⁻¹'*one_Σ⁻¹Σₒ*Σ⁻¹*F⨉I_A⁻¹ - mul!(gradient, ∇A', vec(C*mul!(similar(C), S, I_A⁻¹')), 2, 0) + C = mul!(semml.varXvar_1, F⨉I_A⁻¹', + mul!(semml.obsXvar_1, + Symmetric(mul!(semml.obsXobs_3, one_Σ⁻¹Σₒ, Σ⁻¹)), F⨉I_A⁻¹)) + mul!(gradient, ∇A', + vec(mul!(semml.varXvar_3, + Symmetric(C), + mul!(semml.varXvar_2, S, I_A⁻¹'))), 2, 0) mul!(gradient, ∇S', vec(C), 1, 1) if MeanStructure(implied) === HasMeanStructure @@ -196,8 +203,8 @@ function evaluate!( μ₋ᵀΣ⁻¹ = μ₋'*Σ⁻¹ k = μ₋ᵀΣ⁻¹*F⨉I_A⁻¹ mul!(gradient, ∇M', k', -2, 1) - mul!(gradient, ∇A', vec(k'*(I_A⁻¹*(M + S*k'))'), -2, 1) - mul!(gradient, ∇S', vec(k'k), -1, 1) + mul!(gradient, ∇A', vec(mul!(semml.varXvar_1, k', (I_A⁻¹*(M + S*k'))')), -2, 1) + mul!(gradient, ∇S', vec(mul!(semml.varXvar_2, k', k)), -1, 1) end end diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index fd56ab68c..73da6b81d 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -11,7 +11,7 @@ imply_ram = RAM(specification = spec) imply_ram_sym = RAMSymbolic(specification = spec) # loss functions --------------------------------------------------------------------------- -ml = SemML(observed = observed) +ml = SemML(specification = spec, observed = observed) wls = SemWLS(observed = observed) @@ -185,7 +185,7 @@ imply_ram = RAM(specification = spec_mean, meanstructure = true) imply_ram_sym = RAMSymbolic(specification = spec_mean, meanstructure = true) # loss functions --------------------------------------------------------------------------- -ml = SemML(observed = observed, meanstructure = true) +ml = SemML(observed = observed, specification = spec_mean, meanstructure = true) wls = SemWLS(observed = observed, meanstructure = true) diff --git a/test/examples/recover_parameters/recover_parameters_twofact.jl b/test/examples/recover_parameters/recover_parameters_twofact.jl index 7c6270583..efb0197b4 100644 --- a/test/examples/recover_parameters/recover_parameters_twofact.jl +++ b/test/examples/recover_parameters/recover_parameters_twofact.jl @@ -51,7 +51,7 @@ Random.seed!(1234) x = transpose(rand(true_dist, 100000)) semobserved = SemObservedData(data = x, specification = nothing) -loss_ml = SemLoss(SemML(;observed = semobserved, nparams = length(start))) +loss_ml = SemLoss(SemML(; observed = semobserved, specification = ram_matrices, nparams = length(start))) optimizer = SemOptimizerOptim( From 5edefabc23e68359583e0fc2282c797ae4f64aa1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:06:32 -0700 Subject: [PATCH 143/174] lower/upper_bounds() API for optim --- src/optimizer/documentation.jl | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/optimizer/documentation.jl b/src/optimizer/documentation.jl index 34b78c7f9..eda42a91f 100644 --- a/src/optimizer/documentation.jl +++ b/src/optimizer/documentation.jl @@ -52,3 +52,42 @@ function prepare_start_params(start_val, model::AbstractSem; kwargs...) @assert length(start_val) == nparams(model) return start_val end + +# define a vector of parameter lower bounds: use user-specified vector as is +function lower_bounds(bounds::AbstractVector, model::AbstractSem; + default::Number, variance_default::Number) + length(bound) == nparams(model) || + throw(DimensionMismatch("The length of `bounds` vector ($(length(bounds))) does not match the number of model parameters ($(nparams(model))).")) + return bounds +end + +# define a vector of parameter lower bounds given a dictionary and default values +function lower_bounds(bounds::Union{AbstractDict, Nothing}, model::AbstractSem; + default::Number, variance_default::Number) + varparams = Set(variance_params(model.imply.ram_matrices)) + res = [begin + def = in(p, varparams) ? variance_default : default + isnothing(bounds) ? def : get(bounds, p, def) + end for p in SEM.params(model)] + + return res +end + +# define a vector of parameter upper bounds: use user-specified vector as is +function upper_bounds(bounds::AbstractVector, model::AbstractSem; + default::Number) + length(bound) == nparams(model) || + throw(DimensionMismatch("The length of `bounds` vector ($(length(bounds))) does not match the number of model parameters ($(nparams(model))).")) + return bounds +end + +# define a vector of parameter lower bounds given a dictionary and default values +function upper_bounds(bounds::Union{AbstractDict, Nothing}, model::AbstractSem; + default::Number) + res = [begin + def = default + isnothing(bounds) ? def : get(bounds, p, def) + end for p in SEM.params(model)] + + return res +end From c0f54ee8e70c1e6fd130b5abe2edd0eae7c6979f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:06:52 -0700 Subject: [PATCH 144/174] u/l_bounds support for Optim.jl --- src/optimizer/optim.jl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/optimizer/optim.jl b/src/optimizer/optim.jl index bbb0137d5..b918372ad 100644 --- a/src/optimizer/optim.jl +++ b/src/optimizer/optim.jl @@ -21,13 +21,29 @@ function sem_fit( optim::SemOptimizerOptim, model::AbstractSem, start_params::AbstractVector; + lower_bounds::Union{AbstractVector, AbstractDict, Nothing} = nothing, + upper_bounds::Union{AbstractVector, AbstractDict, Nothing} = nothing, + variance_lower_bound::Float64 = 0.0, + lower_bound = -Inf, + upper_bound = Inf, kwargs...) - result = Optim.optimize( + # setup lower/upper bounds if the algorithm supports it + if optim.algorithm isa Optim.Fminbox || optim.algorithm isa Optim.SAMIN + lbounds = SEM.lower_bounds(lower_bounds, model, default=lower_bound, variance_default=variance_lower_bound) + ubounds = SEM.upper_bounds(upper_bounds, model, default=upper_bound) + result = Optim.optimize( + Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), + lbounds, ubounds, start_params, + model.optimizer.algorithm, + model.optimizer.options) + else + result = Optim.optimize( Optim.only_fgh!((F, G, H, par) -> evaluate!(F, G, H, model, par)), start_params, model.optimizer.algorithm, model.optimizer.options) + end return SemFit(result, model, start_params) end From a73d52db728493b46085e4e2e260657dab6e66ca Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 12 Mar 2024 02:57:20 -0700 Subject: [PATCH 145/174] SemOptimizer(engine = ...) ctor --- src/diff/Empty.jl | 4 ++-- src/diff/NLopt.jl | 4 +++- src/diff/optim.jl | 11 +++++++---- src/types.jl | 13 ++++++++++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/diff/Empty.jl b/src/diff/Empty.jl index 0780907e0..601c34b26 100644 --- a/src/diff/Empty.jl +++ b/src/diff/Empty.jl @@ -15,13 +15,13 @@ an optimizer part. Subtype of `SemOptimizer`. """ -struct SemOptimizerEmpty <: SemOptimizer end +struct SemOptimizerEmpty <: SemOptimizer{:Empty} end ############################################################################################ ### Constructor ############################################################################################ -# SemOptimizerEmpty(;kwargs...) = SemOptimizerEmpty() +SemOptimizer{:Empty}() = SemOptimizerEmpty() ############################################################################################ ### Recommended methods diff --git a/src/diff/NLopt.jl b/src/diff/NLopt.jl index e1f839e79..bf92b1163 100644 --- a/src/diff/NLopt.jl +++ b/src/diff/NLopt.jl @@ -56,7 +56,7 @@ see [Constrained optimization](@ref) in our online documentation. Subtype of `SemOptimizer`. """ -struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer +struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} algorithm::A local_algorithm::A2 options::B @@ -95,6 +95,8 @@ function SemOptimizerNLopt(; inequality_constraints) end +SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) + ############################################################################################ ### Recommended methods ############################################################################################ diff --git a/src/diff/optim.jl b/src/diff/optim.jl index 93695d17c..c1542369b 100644 --- a/src/diff/optim.jl +++ b/src/diff/optim.jl @@ -44,15 +44,18 @@ my_newton_optimizer = SemOptimizerOptim( Subtype of `SemOptimizer`. """ -mutable struct SemOptimizerOptim{A, B} <: SemOptimizer +mutable struct SemOptimizerOptim{A, B} <: SemOptimizer{:Optim} algorithm::A options::B end +SemOptimizer{:Optim}(args...; kwargs...) = + SemOptimizerOptim(args...; kwargs...) + SemOptimizerOptim(; - algorithm = LBFGS(), - options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), - kwargs...) = + algorithm = LBFGS(), + options = Optim.Options(;f_tol = 1e-10, x_tol = 1.5e-8), + kwargs...) = SemOptimizerOptim(algorithm, options) ############################################################################################ diff --git a/src/types.jl b/src/types.jl index d1392e300..6fbdbb5a8 100644 --- a/src/types.jl +++ b/src/types.jl @@ -84,7 +84,18 @@ Supertype of all objects that can serve as the `optimizer` field of a SEM. Connects the SEM to its optimization backend and controls options like the optimization algorithm. If you want to connect the SEM package to a new optimization backend, you should implement a subtype of SemOptimizer. """ -abstract type SemOptimizer end +abstract type SemOptimizer{E} end + +engine(::Type{SemOptimizer{E}}) where E = E +engine(optimizer::SemOptimizer) = engine(typeof(optimizer)) + +SemOptimizer(args...; engine::Symbol = :Optim, kwargs...) = + SemOptimizer{engine}(args...; kwargs...) + +# fallback optimizer constructor +function SemOptimizer{E}(args...; kwargs...) where E + throw(ErrorException("$E optimizer is not supported.")) +end """ Supertype of all objects that can serve as the observed field of a SEM. From e6b17bd29376586984eb38897dee7f92935434a0 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 12 Mar 2024 03:00:29 -0700 Subject: [PATCH 146/174] SEMNLOptExt for NLopt --- Project.toml | 7 ++++- ext/SEMNLOptExt.jl | 12 ++++++++ {src => ext}/diff/NLopt.jl | 49 +++++++++++++++++---------------- {src => ext}/optimizer/NLopt.jl | 8 +++--- src/StructuralEquationModels.jl | 6 ++-- 5 files changed, 50 insertions(+), 32 deletions(-) create mode 100644 ext/SEMNLOptExt.jl rename {src => ext}/diff/NLopt.jl (71%) rename {src => ext}/optimizer/NLopt.jl (93%) diff --git a/Project.toml b/Project.toml index 91f5e8f8c..e9b678408 100644 --- a/Project.toml +++ b/Project.toml @@ -12,7 +12,6 @@ LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NLSolversBase = "d41bc354-129a-5804-8e4c-c37616107c6c" -NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" @@ -42,3 +41,9 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test"] + +[weakdeps] +NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" + +[extensions] +SEMNLOptExt = "NLopt" diff --git a/ext/SEMNLOptExt.jl b/ext/SEMNLOptExt.jl new file mode 100644 index 000000000..dfc3bbb42 --- /dev/null +++ b/ext/SEMNLOptExt.jl @@ -0,0 +1,12 @@ +module SEMNLOptExt + +using StructuralEquationModels, NLopt + +SEM = StructuralEquationModels + +export SemOptimizerNLopt, NLoptConstraint + +include("diff/NLopt.jl") +include("optimizer/NLopt.jl") + +end diff --git a/src/diff/NLopt.jl b/ext/diff/NLopt.jl similarity index 71% rename from src/diff/NLopt.jl rename to ext/diff/NLopt.jl index bf92b1163..043c52934 100644 --- a/src/diff/NLopt.jl +++ b/ext/diff/NLopt.jl @@ -9,10 +9,10 @@ Connects to `NLopt.jl` as the optimization backend. SemOptimizerNLopt(; algorithm = :LD_LBFGS, options = Dict{Symbol, Any}(), - local_algorithm = nothing, - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), + local_algorithm = nothing, + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), kwargs...) # Arguments @@ -37,9 +37,9 @@ my_constrained_optimizer = SemOptimizerNLopt(; ``` # Usage -All algorithms and options from the NLopt library are available, for more information see +All algorithms and options from the NLopt library are available, for more information see the NLopt.jl package and the NLopt online documentation. -For information on how to use inequality and equality constraints, +For information on how to use inequality and equality constraints, see [Constrained optimization](@ref) in our online documentation. # Extended help @@ -65,51 +65,54 @@ struct SemOptimizerNLopt{A, A2, B, B2, C} <: SemOptimizer{:NLopt} inequality_constraints::C end -Base.@kwdef mutable struct NLoptConstraint +Base.@kwdef struct NLoptConstraint f tol = 0.0 end +Base.convert(::Type{NLoptConstraint}, tuple::NamedTuple{(:f, :tol), Tuple{F, T}}) where {F, T} = + NLoptConstraint(tuple.f, tuple.tol) + ############################################################################################ ### Constructor ############################################################################################ function SemOptimizerNLopt(; algorithm = :LD_LBFGS, - local_algorithm = nothing, + local_algorithm = nothing, options = Dict{Symbol, Any}(), - local_options = Dict{Symbol, Any}(), - equality_constraints = Vector{NLoptConstraint}(), - inequality_constraints = Vector{NLoptConstraint}(), + local_options = Dict{Symbol, Any}(), + equality_constraints = Vector{NLoptConstraint}(), + inequality_constraints = Vector{NLoptConstraint}(), kwargs...) - applicable(iterate, equality_constraints) || + applicable(iterate, equality_constraints) && !isa(equality_constraints, NamedTuple) || (equality_constraints = [equality_constraints]) - applicable(iterate, inequality_constraints) || + applicable(iterate, inequality_constraints) && !isa(inequality_constraints, NamedTuple) || (inequality_constraints = [inequality_constraints]) return SemOptimizerNLopt( - algorithm, - local_algorithm, - options, - local_options, - equality_constraints, - inequality_constraints) + algorithm, + local_algorithm, + options, + local_options, + convert.(NLoptConstraint, equality_constraints), + convert.(NLoptConstraint, inequality_constraints)) end -SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) +SEM.SemOptimizer{:NLopt}(args...; kwargs...) = SemOptimizerNLopt(args...; kwargs...) ############################################################################################ ### Recommended methods ############################################################################################ -update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = optimizer +SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs...) = optimizer ############################################################################################ ### additional methods ############################################################################################ -algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm +SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm -options(optimizer::SemOptimizerNLopt) = optimizer.options +SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints diff --git a/src/optimizer/NLopt.jl b/ext/optimizer/NLopt.jl similarity index 93% rename from src/optimizer/NLopt.jl rename to ext/optimizer/NLopt.jl index 2dc9a070f..c760d1b61 100644 --- a/src/optimizer/NLopt.jl +++ b/ext/optimizer/NLopt.jl @@ -7,9 +7,9 @@ mutable struct NLoptResult problem end -optimizer(res::NLoptResult) = res.problem.algorithm -n_iterations(res::NLoptResult) = res.problem.numevals -convergence(res::NLoptResult) = res.result[3] +SEM.optimizer(res::NLoptResult) = res.problem.algorithm +SEM.n_iterations(res::NLoptResult) = res.problem.numevals +SEM.convergence(res::NLoptResult) = res.result[3] # construct SemFit from fitted NLopt object function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) @@ -23,7 +23,7 @@ function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt) end # sem_fit method -function sem_fit( +function SEM.sem_fit( optimizer::SemOptimizerNLopt, model::AbstractSem, start_params::AbstractVector; diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index ca088094b..f1304db72 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -2,7 +2,7 @@ module StructuralEquationModels using LinearAlgebra, Optim, NLSolversBase, Statistics, StatsBase, SparseArrays, Symbolics, - NLopt, FiniteDiff, PrettyTables, + FiniteDiff, PrettyTables, Distributions, StenoGraphs, LazyArtifacts, DelimitedFiles, DataFrames @@ -44,12 +44,10 @@ include("loss/WLS/WLS.jl") include("loss/constant/constant.jl") # optimizer include("diff/optim.jl") -include("diff/NLopt.jl") include("diff/Empty.jl") # optimizer include("optimizer/documentation.jl") include("optimizer/optim.jl") -include("optimizer/NLopt.jl") # helper functions include("additional_functions/helper.jl") include("additional_functions/start_val/start_fabin3.jl") @@ -88,7 +86,7 @@ export AbstractSem, SemLossFunction, SemML, SemFIML, em_mvn, SemLasso, SemRidge, SemConstant, SemWLS, loss, SemOptimizer, - SemOptimizerEmpty, SemOptimizerOptim, SemOptimizerNLopt, NLoptConstraint, + SemOptimizerEmpty, SemOptimizerOptim, optimizer, n_iterations, convergence, SemObserved, SemObservedData, SemObservedCovariance, SemObservedMissing, observed, From 75495affc205919ca0e646fead32caf974dee6ea Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 12 Mar 2024 03:00:57 -0700 Subject: [PATCH 147/174] SEMProximalOptExt for Proximal opt --- Project.toml | 4 ++ ext/SEMProximalOptExt.jl | 15 +++++++ ext/diff/Proximal.jl | 34 +++++++++++++++ ext/optimizer/ProximalAlgorithms.jl | 64 +++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 ext/SEMProximalOptExt.jl create mode 100644 ext/diff/Proximal.jl create mode 100644 ext/optimizer/ProximalAlgorithms.jl diff --git a/Project.toml b/Project.toml index e9b678408..a944d0285 100644 --- a/Project.toml +++ b/Project.toml @@ -44,6 +44,10 @@ test = ["Test"] [weakdeps] NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" +ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" +ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" +ProximalOperators = "f3b72e0c-5f3e-4b3e-8f3e-3f4f3e3e3e3e" [extensions] SEMNLOptExt = "NLopt" +SEMProximalOptExt = ["ProximalCore", "ProximalAlgorithms", "ProximalOperators"] diff --git a/ext/SEMProximalOptExt.jl b/ext/SEMProximalOptExt.jl new file mode 100644 index 000000000..fb9f3c410 --- /dev/null +++ b/ext/SEMProximalOptExt.jl @@ -0,0 +1,15 @@ +module SEMProximalOptExt + +using StructuralEquationModels +using ProximalCore, ProximalAlgorithms, ProximalOperators + +export SemOptimizerProximal + +SEM = StructuralEquationModels + +#ProximalCore.prox!(y, f, x, gamma) = ProximalOperators.prox!(y, f, x, gamma) + +include("diff/Proximal.jl") +include("optimizer/ProximalAlgorithms.jl") + +end diff --git a/ext/diff/Proximal.jl b/ext/diff/Proximal.jl new file mode 100644 index 000000000..6341c9fc6 --- /dev/null +++ b/ext/diff/Proximal.jl @@ -0,0 +1,34 @@ +mutable struct SemOptimizerProximal{A, B, C, D} <: SemOptimizer{:Proximal} + algorithm::A + options::B + operator_g::C + operator_h::D +end + +SEM.SemOptimizer{:Proximal}(args...; kwargs...) = + SemOptimizerProximal(args...; kwargs...) + +SemOptimizerProximal(;algorithm = ProximalAlgorithms.PANOC(), options = Dict{Symbol, Any}(), operator_g, operator_h = nothing, kwargs...) = + SemOptimizerProximal(algorithm, options, operator_g, operator_h) + +############################################################################################ +### Recommended methods +############################################################################################ + +SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) = optimizer + +############################################################################################ +### additional methods +############################################################################################ + +SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm +SEM.options(optimizer::SemOptimizerProximal) = optimizer.options + +############################################################################ +### Pretty Printing +############################################################################ + +function Base.show(io::IO, struct_inst::SemOptimizerProximal) + print_type_name(io, struct_inst) + print_field_types(io, struct_inst) +end \ No newline at end of file diff --git a/ext/optimizer/ProximalAlgorithms.jl b/ext/optimizer/ProximalAlgorithms.jl new file mode 100644 index 000000000..509111d2d --- /dev/null +++ b/ext/optimizer/ProximalAlgorithms.jl @@ -0,0 +1,64 @@ +## connect do ProximalAlgorithms.jl as backend +ProximalCore.gradient!(grad, model::AbstractSem, parameters) = objective_gradient!(grad, model::AbstractSem, parameters) + +mutable struct ProximalResult + result +end + +function SEM.sem_fit( + model::AbstractSemSingle{O, I, L, D}; + start_val = start_val, + kwargs...) where {O, I, L, D <: SemOptimizerProximal} + + if !isa(start_val, Vector) + start_val = start_val(model; kwargs...) + end + + if isnothing(model.optimizer.operator_h) + solution, iterations = model.optimizer.algorithm( + x0 = start_val, + f = model, + g = model.optimizer.operator_g + ) + else + solution, iterations = model.optimizer.algorithm( + x0=start_val, + f=model, + g=model.optimizer.operator_g, + h=model.optimizer.operator_h + ) + end + + minimum = objective!(model, solution) + + optimization_result = Dict( + :minimum => minimum, + :iterations => iterations, + :algorithm => model.optimizer.algorithm, + :operator_g => model.optimizer.operator_g) + + isnothing(model.optimizer.operator_h) || + push!(optimization_result, :operator_h => model.optimizer.operator_h) + + return SemFit( + minimum, + solution, + start_val, + model, + ProximalResult(optimization_result) + ) + +end + +############################################################################################ +# pretty printing +############################################################################################ + +function Base.show(io::IO, result::ProximalResult) + print(io, "Minimum: $(round(result.result[:minimum]; digits = 2)) \n") + print(io, "No. evaluations: $(result.result[:iterations]) \n") + print(io, "Operator: $(nameof(typeof(result.result[:operator_g]))) \n") + if haskey(result.result, :operator_h) + print(io, "Second Operator: $(nameof(typeof(result.result[:operator_h]))) \n") + end +end From cabd4dd33fa1ced87476d9691194748c51d62bb2 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 2 Apr 2024 18:33:50 -0700 Subject: [PATCH 148/174] NLopt: minor tweaks --- ext/optimizer/NLopt.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/optimizer/NLopt.jl b/ext/optimizer/NLopt.jl index c760d1b61..f52be96c4 100644 --- a/ext/optimizer/NLopt.jl +++ b/ext/optimizer/NLopt.jl @@ -35,7 +35,7 @@ function SEM.sem_fit( model.optimizer.options, length(start_params)) set_NLopt_constraints!(opt, model.optimizer) - opt.min_objective = (par, G) -> evaluate!(eltype(par), !isnothing(G) && !isempty(G) ? G : nothing, nothing, model, par) + opt.min_objective = (par, G) -> SEM.evaluate!(eltype(par), !isnothing(G) && !isempty(G) ? G : nothing, nothing, model, par) if !isnothing(model.optimizer.local_algorithm) opt_local = construct_NLopt_problem( @@ -58,20 +58,20 @@ end function construct_NLopt_problem(algorithm, options, npar) opt = Opt(algorithm, npar) - for key in keys(options) - setproperty!(opt, key, options[key]) + for (key, val) in pairs(options) + setproperty!(opt, key, val) end return opt end -function set_NLopt_constraints!(opt, optimizer::SemOptimizerNLopt) +function set_NLopt_constraints!(opt::Opt, optimizer::SemOptimizerNLopt) for con in optimizer.inequality_constraints - inequality_constraint!(opt::Opt, con.f, con.tol) + inequality_constraint!(opt, con.f, con.tol) end for con in optimizer.equality_constraints - equality_constraint!(opt::Opt, con.f, con.tol) + equality_constraint!(opt, con.f, con.tol) end end From e9808edeced7e982682559d46060527f0552e9e2 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 12 Mar 2024 10:53:03 -0700 Subject: [PATCH 149/174] add PackageExtensionCompat --- Project.toml | 1 + src/StructuralEquationModels.jl | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a944d0285..8b698a76d 100644 --- a/Project.toml +++ b/Project.toml @@ -13,6 +13,7 @@ LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" NLSolversBase = "d41bc354-129a-5804-8e4c-c37616107c6c" Optim = "429524aa-4258-5aef-a3af-852621145aeb" +PackageExtensionCompat = "65ce6f38-6b18-4e1d-a461-8949797d7930" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index f1304db72..392384d5f 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -4,7 +4,8 @@ using LinearAlgebra, Optim, NLSolversBase, Statistics, StatsBase, SparseArrays, Symbolics, FiniteDiff, PrettyTables, Distributions, StenoGraphs, LazyArtifacts, DelimitedFiles, - DataFrames + DataFrames, + PackageExtensionCompat export StenoGraphs, @StenoGraph, meld @@ -109,4 +110,9 @@ export AbstractSem, example_data, swap_observed, update_observed, @StenoGraph, →, ←, ↔, ⇔ + +function __init__() + @require_extensions +end + end \ No newline at end of file From d968a3a66568e1b0390254b78a183c7d92087533 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 12 Mar 2024 16:49:33 -0700 Subject: [PATCH 150/174] tests helper: is_extended_tests() to consolidate ENV variable check --- test/examples/helper.jl | 4 ++++ test/examples/political_democracy/political_democracy.jl | 6 +++--- test/runtests.jl | 4 ---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/examples/helper.jl b/test/examples/helper.jl index efab6e7b5..e1f20d299 100644 --- a/test/examples/helper.jl +++ b/test/examples/helper.jl @@ -1,5 +1,9 @@ using LinearAlgebra: norm +function is_extended_tests() + return lowercase(get(ENV, "JULIA_EXTENDED_TESTS", "false")) == "true" +end + function test_gradient(model, parameters; rtol = 1e-10, atol = 0) @test nparams(model) == length(parameters) diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 4c893ceae..91a940d7a 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -102,7 +102,7 @@ semoptimizer = SemOptimizerOptim semoptimizer = SemOptimizerNLopt @testset "RAMMatrices | constructor | NLopt" begin include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "RAMMatrices | parts | Optim" begin include("by_parts.jl") end semoptimizer = SemOptimizerNLopt @@ -128,7 +128,7 @@ semoptimizer = SemOptimizerOptim semoptimizer = SemOptimizerNLopt @testset "RAMMatrices → ParameterTable | constructor | NLopt" begin include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "RAMMatrices → ParameterTable | parts | Optim" begin include("by_parts.jl") end semoptimizer = SemOptimizerNLopt @@ -210,7 +210,7 @@ semoptimizer = SemOptimizerOptim semoptimizer = SemOptimizerNLopt @testset "Graph → ParameterTable | constructor | NLopt" begin include("constructor.jl") end -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" +if is_extended_tests() semoptimizer = SemOptimizerOptim @testset "Graph → ParameterTable | parts | Optim" begin include("by_parts.jl") end semoptimizer = SemOptimizerNLopt diff --git a/test/runtests.jl b/test/runtests.jl index 0d401ed30..acf685866 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,7 +2,3 @@ using Test, SafeTestsets @time @safetestset "Unit Tests" begin include("unit_tests/unit_tests.jl") end @time @safetestset "Example Models" begin include("examples/examples.jl") end - -if !haskey(ENV, "JULIA_EXTENDED_TESTS") || ENV["JULIA_EXTENDED_TESTS"] == "true" - -end \ No newline at end of file From 09b86215cb933c40f1934fe12413a3fa1215a164 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Tue, 12 Mar 2024 16:55:33 -0700 Subject: [PATCH 151/174] tests: fix optimizer usage --- test/Project.toml | 3 ++- test/examples/political_democracy/by_parts.jl | 2 +- .../political_democracy/constraints.jl | 15 ++++++++---- .../political_democracy/constructor.jl | 5 +++- .../political_democracy.jl | 24 +++++++++---------- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/test/Project.toml b/test/Project.toml index 035dcb886..80fa2f4fc 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -5,10 +5,11 @@ FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" \ No newline at end of file +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 73da6b81d..7ac2d9c75 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -25,7 +25,7 @@ loss_ml = SemLoss(ml) loss_wls = SemLoss(wls) # optimizer ------------------------------------------------------------------------------------- -optimizer_obj = semoptimizer() +optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- diff --git a/test/examples/political_democracy/constraints.jl b/test/examples/political_democracy/constraints.jl index b6a9477d0..0db8cc5d0 100644 --- a/test/examples/political_democracy/constraints.jl +++ b/test/examples/political_democracy/constraints.jl @@ -1,4 +1,5 @@ # NLopt constraints ------------------------------------------------------------------------ +using NLopt # 1.5*x1 == x2 (aka 1.5*x1 - x2 == 0) #= function eq_constraint(x, grad) @@ -20,12 +21,15 @@ function ineq_constraint(x, grad) 0.6 - x[30]*x[31] end -constrained_optimizer = SemOptimizerNLopt(; +constrained_optimizer = SemOptimizer(; + engine = :NLopt, algorithm = :AUGLAG, local_algorithm = :LD_LBFGS, - options = Dict(:xtol_rel => 1e-4), - # equality_constraints = NLoptConstraint(;f = eq_constraint, tol = 1e-14), - inequality_constraints = NLoptConstraint(;f = ineq_constraint, tol = 1e-8), + options = Dict( + :xtol_rel => 1e-4 + ), + # equality_constraints = (f = eq_constraint, tol = 1e-14), + inequality_constraints = (f = ineq_constraint, tol = 0.0), ) model_ml_constrained = Sem( @@ -41,7 +45,8 @@ solution_constrained = sem_fit(model_ml_constrained) model_ml_maxeval = Sem( specification = spec, data = dat, - optimizer = SemOptimizerNLopt, + optimizer = SemOptimizer, + engine = :NLopt, options = Dict(:maxeval => 10) ) diff --git a/test/examples/political_democracy/constructor.jl b/test/examples/political_democracy/constructor.jl index 095db75f4..0853aeaad 100644 --- a/test/examples/political_democracy/constructor.jl +++ b/test/examples/political_democracy/constructor.jl @@ -1,9 +1,12 @@ using Statistics: cov, mean +using NLopt ############################################################################################ ### models w.o. meanstructure ############################################################################################ +semoptimizer = SemOptimizer(engine = opt_engine) + model_ml = Sem( specification = spec, data = dat, @@ -148,7 +151,7 @@ end ### test hessians ############################################################################################ -if semoptimizer == SemOptimizerOptim +if opt_engine == :Optim using Optim, LineSearches model_ls = Sem( diff --git a/test/examples/political_democracy/political_democracy.jl b/test/examples/political_democracy/political_democracy.jl index 91a940d7a..dfef56c55 100644 --- a/test/examples/political_democracy/political_democracy.jl +++ b/test/examples/political_democracy/political_democracy.jl @@ -96,16 +96,16 @@ partable_mean = ParameterTable(spec_mean) start_test = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3)] start_test_mean = [fill(1.0, 11); fill(0.05, 3); fill(0.05, 6); fill(0.5, 8); fill(0.05, 3); fill(0.1, 7)] -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "RAMMatrices | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "RAMMatrices | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "RAMMatrices | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "RAMMatrices | parts | NLopt" begin include("by_parts.jl") end end @@ -123,15 +123,15 @@ spec_mean = ParameterTable(spec_mean) partable = spec partable_mean = spec_mean -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "RAMMatrices → ParameterTable | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "RAMMatrices → ParameterTable | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "RAMMatrices → ParameterTable | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "RAMMatrices → ParameterTable | parts | NLopt" begin include("by_parts.jl") end end @@ -205,14 +205,14 @@ partable_mean = spec_mean start_test = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 9)] start_test_mean = [fill(0.5, 8); fill(0.05, 3); fill(1.0, 11); fill(0.05, 3); fill(0.05, 13)] -semoptimizer = SemOptimizerOptim +opt_engine = :Optim @testset "Graph → ParameterTable | constructor | Optim" begin include("constructor.jl") end -semoptimizer = SemOptimizerNLopt +opt_engine = :NLopt @testset "Graph → ParameterTable | constructor | NLopt" begin include("constructor.jl") end if is_extended_tests() - semoptimizer = SemOptimizerOptim + opt_engine = :Optim @testset "Graph → ParameterTable | parts | Optim" begin include("by_parts.jl") end - semoptimizer = SemOptimizerNLopt + opt_engine = :NLopt @testset "Graph → ParameterTable | parts | NLopt" begin include("by_parts.jl") end end From dfb79449c110f660c5f51216bd5919e1b460efcb Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 25 Mar 2024 23:50:32 -0700 Subject: [PATCH 152/174] variance_params(SEMSpec) --- src/frontend/specification/ParameterTable.jl | 8 ++++++++ src/frontend/specification/RAMMatrices.jl | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index fbfac1fc4..ad887b744 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -363,6 +363,14 @@ function update_se_hessian!( return update_partable!(partable, :se, params(fit), se) end +function variance_params(partable::ParameterTable) + res = [param for (param, rel, from, to) in + zip(partable.columns.param, partable.columns.relation, + partable.columns.from, partable.columns.to) + if (rel == :↔) && (from == to)] + unique!(res) +end + """ param_values!(out::AbstractVector, partable::ParameterTable, col::Symbol = :estimate) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index b842708b6..16767c369 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -49,6 +49,17 @@ function latent_vars(ram::RAMMatrices) end end +function variance_params(ram::RAMMatrices) + S_diaginds = Set(diagind(ram.S)) + varparams = Vector{Symbol}() + for (i, param) in enumerate(ram.params) + if any(∈(S_diaginds), param_occurences(ram.S, i)) + push!(varparams, param) + end + end + return unique!(varparams) +end + ############################################################################################ ### Constructor ############################################################################################ From 861733c4943dfe2cf8a4a02e942a74b5c1ade967 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 22:49:34 -0700 Subject: [PATCH 153/174] nonunique() helper function --- src/additional_functions/helper.jl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 9335e17b6..fc7a124f4 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -251,4 +251,19 @@ function commutation_matrix_pre_square_add_mt!(B, A) # comuptes B + KₙA # 0 al return B -end \ No newline at end of file +end + +# returns the vector of non-unique values in the order of appearance +# each non-unique values is reported once +function nonunique(values::AbstractVector) + value_counts = Dict{eltype(values), Int}() + res = similar(values, 0) + for v in values + n = get!(value_counts, v, 0) + if n == 1 # second encounter + push!(res, v) + end + value_counts[v] = n + 1 + end + return res +end From 74561b32d698b79a95b88002f4a242cebc8ffc4b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 22:51:07 -0700 Subject: [PATCH 154/174] RAMMatrices ctor: dupl. vars check --- src/frontend/specification/RAMMatrices.jl | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/frontend/specification/RAMMatrices.jl b/src/frontend/specification/RAMMatrices.jl index 16767c369..db481c2ce 100644 --- a/src/frontend/specification/RAMMatrices.jl +++ b/src/frontend/specification/RAMMatrices.jl @@ -71,6 +71,8 @@ function RAMMatrices(; A::AbstractMatrix, S::AbstractMatrix, ncols = size(A, 2) if !isnothing(colnames) length(colnames) == ncols || throw(DimensionMismatch("colnames length ($(length(colnames))) does not match the number of columns in A ($ncols)")) + dup_cols = nonunique(colnames) + isempty(dup_cols) || throw(ArgumentError("Duplicate variables detected: $(join(dup_cols, ", "))")) end size(A, 1) == size(A, 2) || throw(DimensionMismatch("A must be a square matrix")) size(S, 1) == size(S, 2) || throw(DimensionMismatch("S must be a square matrix")) @@ -80,6 +82,9 @@ function RAMMatrices(; A::AbstractMatrix, S::AbstractMatrix, if !isnothing(M) length(M) == ncols || throw(DimensionMismatch("M should have as many elements as colnames length ($ncols), $(length(M)) found")) end + dup_params = nonunique(params) + isempty(dup_params) || throw(ArgumentError("Duplicate parameters detected: $(join(dup_params, ", "))")) + A = ParamsMatrix{Float64}(A, params) S = ParamsMatrix{Float64}(S, params) M = !isnothing(M) ? ParamsVector{Float64}(M, params) : nothing @@ -98,16 +103,10 @@ function RAMMatrices(partable::ParameterTable; params::Union{AbstractVector{Symbol}, Nothing} = nothing) params = copy(isnothing(params) ? SEM.params(partable) : params) - params_index = Dict(param => i for (i, param) in enumerate(params)) - if length(params) != length(params_index) - params_seen = Set{Symbol}() - params_nonunique = Vector{Symbol}() - for par in params - push!(par in params_seen ? params_nonunique : params_seen, par) - end - throw(ArgumentError("Duplicate names in the parameters vector: $(join(params_nonunique, ", "))")) - end + dup_params = nonunique(params) + isempty(dup_params) || throw(ArgumentError("Duplicate parameters detected: $(join(dup_params, ", "))")) + params_index = Dict(param => i for (i, param) in enumerate(params)) n_observed = length(partable.variables.observed) n_latent = length(partable.variables.latent) n_vars = n_observed + n_latent From a46f0eddfb40e1a6e9e9872214264c482bfcdb71 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 22:50:18 -0700 Subject: [PATCH 155/174] ParTable: better params unique check --- src/frontend/specification/ParameterTable.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index ad887b744..05eceb75a 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -290,10 +290,9 @@ function update_partable!(partable::ParameterTable, default::Any = nothing) length(params) == length(values) || throw(ArgumentError("The length of `params` ($(length(params))) and their `values` ($(length(values))) must be the same")) + dup_params = nonunique(params) + isempty(dup_params) || throw(ArgumentError("Duplicate parameters detected: $(join(dup_params, ", "))")) params_dict = Dict(zip(params, values)) - if length(params_dict) != length(params) - throw(ArgumentError("Duplicate parameter names in `params`")) - end update_partable!(partable, column, params_dict, default) end From a26a1657cd55ab5d4ff1a8702e96f577d79eb13c Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:01:19 -0700 Subject: [PATCH 156/174] RAMSymbolic: calc (I-A)^{-1} once --- src/imply/RAM/symbolic.jl | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/imply/RAM/symbolic.jl b/src/imply/RAM/symbolic.jl index d32157c19..7be51f139 100644 --- a/src/imply/RAM/symbolic.jl +++ b/src/imply/RAM/symbolic.jl @@ -109,8 +109,10 @@ function RAMSymbolic(; vech = true end + I_A⁻¹ = neumann_series(A) + # Σ - Σ_symbolic = get_Σ_symbolic_RAM(S, A, F; vech = vech) + Σ_symbolic = eval_Σ_symbolic(S, I_A⁻¹, F; vech = vech) #print(Symbolics.build_function(Σ_symbolic)[2]) Σ_function = Symbolics.build_function(Σ_symbolic, par, expression=Val{false})[2] Σ = zeros(size(Σ_symbolic)) @@ -150,7 +152,7 @@ function RAMSymbolic(; # μ if meanstructure MS = HasMeanStructure - μ_symbolic = get_μ_symbolic_RAM(M, A, F) + μ_symbolic = eval_μ_symbolic(M, I_A⁻¹, F) μ_function = Symbolics.build_function(μ_symbolic, par, expression=Val{false})[2] μ = zeros(size(μ_symbolic)) if gradient @@ -224,24 +226,24 @@ end ### additional functions ############################################################################################ -function get_Σ_symbolic_RAM(S, A, F; vech = false) - invia = neumann_series(A) - Σ_symbolic = F*invia*S*permutedims(invia)*permutedims(F) - Σ_symbolic = Array(Σ_symbolic) - # Σ_symbolic = Symbolics.simplify.(Σ_symbolic) - if vech Σ_symbolic = Σ_symbolic[tril(trues(size(F, 1), size(F, 1)))] end - Threads.@threads for i in eachindex(Σ_symbolic) - Σ_symbolic[i] = Symbolics.simplify(Σ_symbolic[i]) +# expected covariations of observed vars +function eval_Σ_symbolic(S, I_A⁻¹, F; vech = false) + Σ = F*I_A⁻¹*S*permutedims(I_A⁻¹)*permutedims(F) + Σ = Array(Σ) + vech && (Σ = Σ[tril(trues(size(F, 1), size(F, 1)))]) + # Σ = Symbolics.simplify.(Σ) + Threads.@threads for i in eachindex(Σ) + Σ[i] = Symbolics.simplify(Σ[i]) end - return Σ_symbolic + return Σ end -function get_μ_symbolic_RAM(M, A, F) - invia = neumann_series(A) - μ_symbolic = F*invia*M - μ_symbolic = Array(μ_symbolic) - Threads.@threads for i in eachindex(μ_symbolic) - μ_symbolic[i] = Symbolics.simplify(μ_symbolic[i]) +# expected means of observed vars +function eval_μ_symbolic(M, I_A⁻¹, F) + μ = F*I_A⁻¹*M + μ = Array(μ) + Threads.@threads for i in eachindex(μ) + μ[i] = Symbolics.simplify(μ[i]) end - return μ_symbolic + return μ end \ No newline at end of file From db2344c4cd88eaba2a99591950d55cde85a473b1 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:02:29 -0700 Subject: [PATCH 157/174] AbstractSemSingle: vars API --- src/frontend/specification/Sem.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/frontend/specification/Sem.jl b/src/frontend/specification/Sem.jl index 0c22db1d8..5b024a809 100644 --- a/src/frontend/specification/Sem.jl +++ b/src/frontend/specification/Sem.jl @@ -20,6 +20,14 @@ function Sem(; return sem end +nvars(sem::AbstractSemSingle) = nvars(sem.imply.ram_matrices) +nobserved_vars(sem::AbstractSemSingle) = nobserved_vars(sem.imply.ram_matrices) +nlatent_vars(sem::AbstractSemSingle) = nlatent_vars(sem.imply.ram_matrices) + +vars(sem::AbstractSemSingle) = vars(sem.imply.ram_matrices) +observed_vars(sem::AbstractSemSingle) = observed_vars(sem.imply.ram_matrices) +latent_vars(sem::AbstractSemSingle) = latent_vars(sem.imply.ram_matrices) + function SemFiniteDiff(; observed::O = SemObservedData, imply::I = RAM, From e8fb87827af5d5019e5d1f00c6e982ccb07f2343 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 00:13:49 -0700 Subject: [PATCH 158/174] predict_latent_vars() --- src/StructuralEquationModels.jl | 1 + src/frontend/predict.jl | 94 +++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 src/frontend/predict.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 392384d5f..90e436cf4 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -23,6 +23,7 @@ include("frontend/specification/RAMMatrices.jl") include("frontend/specification/EnsembleParameterTable.jl") include("frontend/specification/StenoGraphs.jl") include("frontend/fit/summary.jl") +include("frontend/predict.jl") # pretty printing include("frontend/pretty_printing.jl") # observed diff --git a/src/frontend/predict.jl b/src/frontend/predict.jl new file mode 100644 index 000000000..a2c50e473 --- /dev/null +++ b/src/frontend/predict.jl @@ -0,0 +1,94 @@ +abstract type SemScoresPredictMethod end + +struct SemRegressionScores <: SemScoresPredictMethod end +struct SemBartlettScores <: SemScoresPredictMethod end +struct SemAndersonRubinScores <: SemScoresPredictMethod end + +function SemScoresPredictMethod(method::Symbol) + if method == :regression + return SemRegressionScores() + elseif method == :Bartlett + return SemBartlettScores() + elseif method == :AndersonRubin + return SemAndersonRubinScores() + else + throw(ArgumentError("Unsupported prediction method: $method")) + end +end + +predict_latent_scores(fit::SemFit, data::SemObserved = fit.model.observed; + method::Symbol = :regression) = + predict_latent_scores(SemScoresPredictMethod(method), fit, data) + +predict_latent_scores(method::SemScoresPredictMethod, fit::SemFit, + data::SemObserved = fit.model.observed) = + predict_latent_scores(method, fit.model, fit.solution, data) + +function inv_cov!(A::AbstractMatrix) + if istril(A) + A = LowerTriangular(A) + elseif istriu(A) + A = UpperTriangular(A) + else + end + A_chol = Cholesky(A) + return inv!(A_chol) +end + +function latent_scores_operator(::SemRegressionScores, model::AbstractSemSingle, params::AbstractVector) + implied = model.imply + ram = implied.ram_matrices + lv_inds = latent_var_indices(ram) + + A = materialize(ram.A, params) + lv_FA = ram.F * A[:, lv_inds] + lv_I_A⁻¹ = inv(I - A)[lv_inds, :] + + S = materialize(ram.S, params) + + cov_lv = lv_I_A⁻¹ * S * lv_I_A⁻¹' + Σ = implied.Σ + Σ⁻¹ = inv(Σ) + return cov_lv * lv_FA' * Σ⁻¹ +end + +function latent_scores_operator(::SemBartlettScores, model::AbstractSemSingle, params::AbstractVector) + implied = model.imply + ram = implied.ram_matrices + lv_inds = latent_var_indices(ram) + A = materialize(ram.A, params) + lv_FA = ram.F * A[:, lv_inds] + + S = materialize(ram.S, params) + obs_inds = observed_var_indices(ram) + ov_S⁻¹ = inv(S[obs_inds, obs_inds]) + + return inv(lv_FA' * ov_S⁻¹ * lv_FA) * lv_FA' * ov_S⁻¹ +end + +function predict_latent_scores(method::SemScoresPredictMethod, model::AbstractSemSingle, params::AbstractVector, data::SemObserved) + n_man(data) == nobserved_vars(model) || + throw(DimensionMismatch("Number of variables in data ($(n_obs(data))) does not match the number of observed variables in the model ($(nobserved_vars(model)))")) + length(params) == nparams(model) || + throw(DimensionMismatch("The length of parameters vector ($(length(params))) does not match the number of parameters in the model ($(nparams(model)))")) + + implied = model.imply + hasmeanstruct = MeanStructure(implied) === HasMeanStructure + + update!(EvaluationTargets(0.0, nothing, nothing), model.imply, model, params) + ram = implied.ram_matrices + lv_inds = latent_var_indices(ram) + A = materialize(ram.A, params) + lv_I_A⁻¹ = inv(I - A)[lv_inds, :] + + lv_scores_op = latent_scores_operator(method, model, params) + + data = data.data .- (isnothing(data.obs_mean) ? mean(data.data, dims=1) : data.obs_mean') + lv_scores = data * lv_scores_op' + if hasmeanstruct + M = materialize(ram.M, params) + lv_scores .+= (lv_I_A⁻¹ * M)' + end + + return lv_scores +end From 36b9274c50f0522891c3fca3d6235c2976272980 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:07:34 -0700 Subject: [PATCH 159/174] lavaan_model() --- src/frontend/specification/ParameterTable.jl | 72 ++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/frontend/specification/ParameterTable.jl b/src/frontend/specification/ParameterTable.jl index 05eceb75a..b76921c7b 100644 --- a/src/frontend/specification/ParameterTable.jl +++ b/src/frontend/specification/ParameterTable.jl @@ -520,3 +520,75 @@ lavaan_param_values(partable_lav, partable::ParameterTable, lav_col::Symbol = :est, lav_group = nothing) = lavaan_param_values!(fill(NaN, nparams(partable)), partable_lav, partable, lav_col, lav_group) + +""" + lavaan_model(partable::ParameterTable) + +Generate lavaan model definition from a `partable`. +""" +function lavaan_model(partable::ParameterTable) + latent_vars = Set(partable.variables.latent) + observed_vars = Set(partable.variables.observed) + + variance_defs = Dict{Symbol, IOBuffer}() + latent_dep_defs = Dict{Symbol, IOBuffer}() + latent_regr_defs = Dict{Symbol, IOBuffer}() + observed_regr_defs = Dict{Symbol, IOBuffer}() + + model = IOBuffer() + for (from, to, rel, param, value, free) in + zip(partable.columns.from, partable.columns.to, + partable.columns.relation, partable.columns.param, + partable.columns.value_fixed, partable.columns.free,) + + function append_param(io) + if free + @assert param != :const + param == Symbol("") || write(io, "$param * ") + else + write(io, "$value * ") + end + end + function append_rhs(io) + if position(io) > 0 + write(io, " + ") + end + append_param(io) + write(io, "$to") + end + + if from == Symbol("1") + write(model, "$to ~ ") + append_param(model) + write(model, "1\n") + else + if rel == :↔ + variance_def = get!(() -> IOBuffer(), variance_defs, from) + append_rhs(variance_def) + elseif rel == :→ + if (from ∈ latent_vars) && (to ∈ observed_vars) + latent_dep_def = get!(() -> IOBuffer(), latent_dep_defs, from) + append_rhs(latent_dep_def) + elseif (from ∈ latent_vars) && (to ∈ latent_vars) + latent_regr_def = get!(() -> IOBuffer(), latent_regr_defs, from) + append_rhs(latent_regr_def) + else + observed_regr_def = get!(() -> IOBuffer(), observed_regr_defs, from) + append_rhs(observed_regr_def) + end + end + end + end + function write_rules(io, defs, relation) + vars = sort!(collect(keys(defs))) + for var in vars + write(io, String(var), " ", relation, " ") + write(io, String(take!(defs[var])), "\n") + end + end + write_rules(model, latent_dep_defs, "=~") + write_rules(model, latent_regr_defs, "~") + write_rules(model, observed_regr_defs, "~") + write_rules(model, variance_defs, "~~") + return String(take!(model)) +end \ No newline at end of file From 0ba79480b52c9c1166a5b175b7fc94b893660040 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Mon, 1 Apr 2024 10:08:28 -0700 Subject: [PATCH 160/174] BlackBoxOptim.jl backend support --- Project.toml | 3 + ext/SEMBlackBoxOptimExt/AdamMutation.jl | 44 ++++++ ext/SEMBlackBoxOptimExt/BlackBoxOptim.jl | 75 ++++++++++ ext/SEMBlackBoxOptimExt/DiffEvoFactory.jl | 138 ++++++++++++++++++ .../SEMBlackBoxOptimExt.jl | 13 ++ .../SemOptimizerBlackBoxOptim.jl | 76 ++++++++++ 6 files changed, 349 insertions(+) create mode 100644 ext/SEMBlackBoxOptimExt/AdamMutation.jl create mode 100644 ext/SEMBlackBoxOptimExt/BlackBoxOptim.jl create mode 100644 ext/SEMBlackBoxOptimExt/DiffEvoFactory.jl create mode 100644 ext/SEMBlackBoxOptimExt/SEMBlackBoxOptimExt.jl create mode 100644 ext/SEMBlackBoxOptimExt/SemOptimizerBlackBoxOptim.jl diff --git a/Project.toml b/Project.toml index 8b698a76d..d9287a44c 100644 --- a/Project.toml +++ b/Project.toml @@ -44,7 +44,9 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" test = ["Test"] [weakdeps] +BlackBoxOptim = "a134a8b2-14d6-55f6-9291-3336d3ab0209" NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" ProximalOperators = "f3b72e0c-5f3e-4b3e-8f3e-3f4f3e3e3e3e" @@ -52,3 +54,4 @@ ProximalOperators = "f3b72e0c-5f3e-4b3e-8f3e-3f4f3e3e3e3e" [extensions] SEMNLOptExt = "NLopt" SEMProximalOptExt = ["ProximalCore", "ProximalAlgorithms", "ProximalOperators"] +SEMBlackBoxOptimExt = ["BlackBoxOptim", "Optimisers"] diff --git a/ext/SEMBlackBoxOptimExt/AdamMutation.jl b/ext/SEMBlackBoxOptimExt/AdamMutation.jl new file mode 100644 index 000000000..8db2c3d42 --- /dev/null +++ b/ext/SEMBlackBoxOptimExt/AdamMutation.jl @@ -0,0 +1,44 @@ +# mutate by moving in the gradient direction +mutable struct AdamMutation{M <: AbstractSem, O, S} <: MutationOperator + model::M + optim::O + opt_state::S + params_fraction::Float64 + + function AdamMutation(model::AbstractSem, params::AbstractDict) + optim = RAdam(params[:AdamMutation_eta], params[:AdamMutation_beta]) + params_fraction = params[:AdamMutation_params_fraction] + opt_state = Optimisers.init(optim, Vector{Float64}(undef, nparams(model))) + + new{typeof(model), typeof(optim), typeof(opt_state)}( + model, optim, opt_state, params_fraction) + end +end + +Base.show(io::IO, op::AdamMutation) = print(io, "AdamMutation(", op.optim, " state[3]=", op.opt_state[3], ")") + +""" +Default parameters for `AdamMutation`. +""" +const AdamMutation_DefaultOptions = ParamsDict( + :AdamMutation_eta => 1E-1, + :AdamMutation_beta => (0.99, 0.999), + :AdamMutation_params_fraction => 0.25, +) + +function BlackBoxOptim.apply!(m::AdamMutation, v::AbstractVector{<:Real}, target_index::Int) + grad = similar(v) + obj = SEM.evaluate!(0.0, grad, nothing, m.model, v) + @inbounds for i in eachindex(grad) + (rand() > m.params_fraction) && (grad[i] = 0.0) + end + + m.opt_state, dv = Optimisers.apply!(m.optim, m.opt_state, v, grad) + if (m.opt_state[3][1] <= 1E-20) || !isfinite(obj) || any(!isfinite, dv) + m.opt_state = Optimisers.init(m.optim, v) + else + v .-= dv + end + + return v +end diff --git a/ext/SEMBlackBoxOptimExt/BlackBoxOptim.jl b/ext/SEMBlackBoxOptimExt/BlackBoxOptim.jl new file mode 100644 index 000000000..a48b5c806 --- /dev/null +++ b/ext/SEMBlackBoxOptimExt/BlackBoxOptim.jl @@ -0,0 +1,75 @@ +############################################################################################ +### connect to BlackBoxOptim.jl as backend +############################################################################################ + +""" +""" +struct SemOptimizerBlackBoxOptim <: SemOptimizer{:BlackBoxOptim} + lower_bound::Float64 # default lower bound + variance_lower_bound::Float64 # default variance lower bound + lower_bounds::Union{Dict{Symbol, Float64}, Nothing} + + upper_bound::Float64 # default upper bound + upper_bounds::Union{Dict{Symbol, Float64}, Nothing} +end + +function SemOptimizerBlackBoxOptim(; + lower_bound::Float64 = -1000.0, + lower_bounds::Union{AbstractDict{Symbol, Float64}, Nothing} = nothing, + variance_lower_bound::Float64 = 0.001, + upper_bound::Float64 = 1000.0, + upper_bounds::Union{AbstractDict{Symbol, Float64}, Nothing} = nothing, + kwargs... + ) + if variance_lower_bound < 0.0 + throw(ArgumentError("variance_lower_bound must be non-negative")) + end + return SemOptimizerBlackBoxOptim(lower_bound, variance_lower_bound, lower_bounds, + upper_bound, upper_bounds) +end + +SEM.SemOptimizer{:BlackBoxOptim}(args...; kwargs...) = SemOptimizerBlackBoxOptim(args...; kwargs...) + +SEM.algorithm(optimizer::SemOptimizerBlackBoxOptim) = optimizer.algorithm +SEM.options(optimizer::SemOptimizerBlackBoxOptim) = optimizer.options + +struct SemModelBlackBoxOptimProblem{M <: AbstractSem} <: OptimizationProblem{ScalarFitnessScheme{true}} + model::M + fitness_scheme::ScalarFitnessScheme{true} + search_space::ContinuousRectSearchSpace +end + +function BlackBoxOptim.search_space(model::AbstractSem) + optim = model.optimizer::SemOptimizerBlackBoxOptim + varparams = Set(SEM.variance_params(model.imply.ram_matrices)) + return ContinuousRectSearchSpace( + [begin + def = in(p, varparams) ? optim.variance_lower_bound : optim.lower_bound + isnothing(optim.lower_bounds) ? def : get(optim.lower_bounds, p, def) + end for p in SEM.params(model)], + [begin + def = optim.upper_bound + isnothing(optim.upper_bounds) ? def : get(optim.upper_bounds, p, def) + end for p in SEM.params(model)]) +end + +function SemModelBlackBoxOptimProblem(model::AbstractSem, optimizer::SemOptimizerBlackBoxOptim) + SemModelBlackBoxOptimProblem(model, ScalarFitnessScheme{true}(), search_space(model)) +end + +BlackBoxOptim.fitness(params::AbstractVector, wrapper::SemModelBlackBoxOptimProblem) = + return SEM.evaluate!(0.0, nothing, nothing, wrapper.model, params) + +# sem_fit method +function SEM.sem_fit( + optimizer::SemOptimizerBlackBoxOptim, + model::AbstractSem, + start_params::AbstractVector; + MaxSteps::Integer = 50000, + kwargs...) + + problem = SemModelBlackBoxOptimProblem(model, optimizer) + res = bboptimize(problem; MaxSteps, kwargs...) + return SemFit(best_fitness(res), best_candidate(res), + nothing, model, res) +end diff --git a/ext/SEMBlackBoxOptimExt/DiffEvoFactory.jl b/ext/SEMBlackBoxOptimExt/DiffEvoFactory.jl new file mode 100644 index 000000000..22ea97b89 --- /dev/null +++ b/ext/SEMBlackBoxOptimExt/DiffEvoFactory.jl @@ -0,0 +1,138 @@ +""" +Base class for factories of optimizers for a specific problem. +""" +abstract type OptimizerFactory{P<:OptimizationProblem} end + +problem(factory::OptimizerFactory) = factory.problem + +const OptController_DefaultParameters = ParamsDict( + :MaxTime => 60.0, :MaxSteps => 10^8, + :TraceMode => :compact, :TraceInterval => 5.0, + :RecoverResults => false, :SaveTrace => false +) + +function generate_opt_controller(alg::Optimizer, optim_factory::OptimizerFactory, params) + return BlackBoxOptim.OptController(alg, problem(optim_factory), + BlackBoxOptim.chain(BlackBoxOptim.DefaultParameters, + OptController_DefaultParameters, + params)) +end + +function check_population(factory::OptimizerFactory, popmatrix::BlackBoxOptim.PopulationMatrix) + ssp = factory |> problem |> search_space + for i in 1:popsize(popmatrix) + @assert popmatrix[:, i] ∈ ssp "Individual $i is out of space: $(popmatrix[:,i])" # fitness: $(fitness(population, i))" + end +end + +initial_search_space(factory::OptimizerFactory, id::Int) = search_space(factory.problem) + +function initial_population_matrix(factory::OptimizerFactory, id::Int) + #@info "Standard initial_population_matrix()" + ini_ss = initial_search_space(factory, id) + if !isempty(factory.initial_population) + numdims(factory.initial_population) == numdims(factory.problem) || + throw(DimensionMismatch("Dimensions of :Population ($(numdims(factory.initial_population))) "* + "are different from the problem dimensions ($(numdims(factory.problem)))")) + res = factory.initial_population[:, StatsBase.sample(1:popsize(factory.initial_population), factory.population_size)] + else + res = rand_individuals(ini_ss, factory.population_size, method=:latin_hypercube) + end + prj = RandomBound(ini_ss) + if size(res, 2) > 1 + apply!(prj, view(res, :, 1), SEM.start_fabin3(factory.problem.model)) + end + if size(res, 2) > 2 + apply!(prj, view(res, :, 2), SEM.start_simple(factory.problem.model)) + end + return res +end + +# convert individuals in the archive into population matrix +population_matrix(archive::Any) = + population_matrix!(Matrix{Float64}(undef, length(BlackBoxOptim.params(first(archive))), length(archive)), + archive) + +function population_matrix!(pop::AbstractMatrix{<:Real}, archive::Any) + npars = length(BlackBoxOptim.params(first(archive))) + size(pop, 1) == npars || + throw(DimensionMismatch("Matrix rows count ($(size(pop, 1))) doesn't match the number of problem dimensions ($(npars))")) + @inbounds for (i, indi) in enumerate(archive) + (i <= size(pop, 2)) || break + pop[:, i] .= BlackBoxOptim.params(indi) + end + if size(pop, 2) > length(archive) + @warn "Matrix columns count ($(size(pop, 2))) is bigger than population size ($(length(archive))), last columns not set" + end + return pop +end + +generate_embedder(factory::OptimizerFactory, id::Int, problem::OptimizationProblem) = + RandomBound(search_space(problem)) + +abstract type DiffEvoFactory{P<:OptimizationProblem} <: OptimizerFactory{P} end + +generate_selector(factory::DiffEvoFactory, id::Int, problem::OptimizationProblem, population) = + RadiusLimitedSelector(get(factory.params, :selector_radius, popsize(population) ÷ 5)) + +function generate_modifier(factory::DiffEvoFactory, id::Int, problem::OptimizationProblem) + ops = GeneticOperator[ + MutationClock(UniformMutation(search_space(problem)), 1/numdims(problem)), + BlackBoxOptim.AdaptiveDiffEvoRandBin1(BlackBoxOptim.AdaptiveDiffEvoParameters(factory.params[:fdistr], factory.params[:crdistr])), + SimplexCrossover{3}(1.05), + SimplexCrossover{2}(1.1), + #SimulatedBinaryCrossover(0.05, 16.0), + #SimulatedBinaryCrossover(0.05, 3.0), + #SimulatedBinaryCrossover(0.1, 5.0), + #SimulatedBinaryCrossover(0.2, 16.0), + UnimodalNormalDistributionCrossover{2}(chain(BlackBoxOptim.UNDX_DefaultOptions, factory.params)), + UnimodalNormalDistributionCrossover{3}(chain(BlackBoxOptim.UNDX_DefaultOptions, factory.params)), + ParentCentricCrossover{2}(chain(BlackBoxOptim.PCX_DefaultOptions, factory.params)), + ParentCentricCrossover{3}(chain(BlackBoxOptim.PCX_DefaultOptions, factory.params)) + ] + if problem isa SemModelBlackBoxOptimProblem + push!(ops, AdamMutation(problem.model, chain(AdamMutation_DefaultOptions, factory.params))) + end + FAGeneticOperatorsMixture(ops) +end + +function generate_optimizer(factory::DiffEvoFactory, id::Int, problem::OptimizationProblem, popmatrix) + population = FitPopulation(popmatrix, nafitness(fitness_scheme(problem))) + BlackBoxOptim.DiffEvoOpt("AdaptiveDE/rand/1/bin/gradient", population, + generate_selector(factory, id, problem, population), + generate_modifier(factory, id, problem), + generate_embedder(factory, id, problem)) +end + +const Population_DefaultParameters = ParamsDict( + :Population => BlackBoxOptim.PopulationMatrix(undef, 0, 0), + :PopulationSize => 100, +) + +const DE_DefaultParameters = chain(ParamsDict( + :SelectorRadius => 0, + :fdistr => BlackBoxOptim.BimodalCauchy(0.65, 0.1, 1.0, 0.1, clampBelow0 = false), + :crdistr => BlackBoxOptim.BimodalCauchy(0.1, 0.1, 0.95, 0.1, clampBelow0 = false), +), Population_DefaultParameters) + +struct DefaultDiffEvoFactory{P<:OptimizationProblem} <: DiffEvoFactory{P} + problem::P + initial_population::BlackBoxOptim.PopulationMatrix + population_size::Int + params::ParamsDictChain +end + +DefaultDiffEvoFactory(problem::OptimizationProblem; kwargs...) = + DefaultDiffEvoFactory(problem, BlackBoxOptim.kwargs2dict(kwargs)) + +function DefaultDiffEvoFactory(problem::OptimizationProblem, params::AbstractDict) + params = chain(DE_DefaultParameters, params) + DefaultDiffEvoFactory{typeof(problem)}(problem, params[:Population], params[:PopulationSize], params) +end + +function BlackBoxOptim.bbsetup(factory::OptimizerFactory; kwargs...) + popmatrix = initial_population_matrix(factory, 1) + check_population(factory, popmatrix) + alg = generate_optimizer(factory, 1, problem(factory), popmatrix) + return generate_opt_controller(alg, factory, BlackBoxOptim.kwargs2dict(kwargs)) +end diff --git a/ext/SEMBlackBoxOptimExt/SEMBlackBoxOptimExt.jl b/ext/SEMBlackBoxOptimExt/SEMBlackBoxOptimExt.jl new file mode 100644 index 000000000..9cbdac4d0 --- /dev/null +++ b/ext/SEMBlackBoxOptimExt/SEMBlackBoxOptimExt.jl @@ -0,0 +1,13 @@ +module SEMBlackBoxOptimExt + +using StructuralEquationModels, BlackBoxOptim, Optimisers + +SEM = StructuralEquationModels + +export SemOptimizerBlackBoxOptim + +include("AdamMutation.jl") +include("DiffEvoFactory.jl") +include("SemOptimizerBlackBoxOptim.jl") + +end diff --git a/ext/SEMBlackBoxOptimExt/SemOptimizerBlackBoxOptim.jl b/ext/SEMBlackBoxOptimExt/SemOptimizerBlackBoxOptim.jl new file mode 100644 index 000000000..824736785 --- /dev/null +++ b/ext/SEMBlackBoxOptimExt/SemOptimizerBlackBoxOptim.jl @@ -0,0 +1,76 @@ +############################################################################################ +### connect to BlackBoxOptim.jl as backend +############################################################################################ + +""" +""" +struct SemOptimizerBlackBoxOptim <: SemOptimizer{:BlackBoxOptim} + lower_bound::Float64 # default lower bound + variance_lower_bound::Float64 # default variance lower bound + lower_bounds::Union{Dict{Symbol, Float64}, Nothing} + + upper_bound::Float64 # default upper bound + upper_bounds::Union{Dict{Symbol, Float64}, Nothing} +end + +function SemOptimizerBlackBoxOptim(; + lower_bound::Float64 = -1000.0, + lower_bounds::Union{AbstractDict{Symbol, Float64}, Nothing} = nothing, + variance_lower_bound::Float64 = 0.001, + upper_bound::Float64 = 1000.0, + upper_bounds::Union{AbstractDict{Symbol, Float64}, Nothing} = nothing, + kwargs... + ) + if variance_lower_bound < 0.0 + throw(ArgumentError("variance_lower_bound must be non-negative")) + end + return SemOptimizerBlackBoxOptim(lower_bound, variance_lower_bound, lower_bounds, + upper_bound, upper_bounds) +end + +SEM.SemOptimizer{:BlackBoxOptim}(args...; kwargs...) = SemOptimizerBlackBoxOptim(args...; kwargs...) + +SEM.algorithm(optimizer::SemOptimizerBlackBoxOptim) = optimizer.algorithm +SEM.options(optimizer::SemOptimizerBlackBoxOptim) = optimizer.options + +struct SemModelBlackBoxOptimProblem{M <: AbstractSem} <: OptimizationProblem{ScalarFitnessScheme{true}} + model::M + fitness_scheme::ScalarFitnessScheme{true} + search_space::ContinuousRectSearchSpace +end + +function BlackBoxOptim.search_space(model::AbstractSem) + optim = model.optimizer::SemOptimizerBlackBoxOptim + return ContinuousRectSearchSpace( + SEM.lower_bounds(optim.lower_bounds, model, default=optim.lower_bound, variance_default=optim.variance_lower_bound), + SEM.upper_bounds(optim.upper_bounds, model, default=optim.upper_bound)) +end + +function SemModelBlackBoxOptimProblem(model::AbstractSem, optimizer::SemOptimizerBlackBoxOptim) + SemModelBlackBoxOptimProblem(model, ScalarFitnessScheme{true}(), search_space(model)) +end + +BlackBoxOptim.fitness(params::AbstractVector, wrapper::SemModelBlackBoxOptimProblem) = + return SEM.evaluate!(0.0, nothing, nothing, wrapper.model, params) + +# sem_fit method +function SEM.sem_fit( + optimizer::SemOptimizerBlackBoxOptim, + model::AbstractSem, + start_params::AbstractVector; + Method::Symbol = :adaptive_de_rand_1_bin_with_gradient, + MaxSteps::Integer = 50000, + kwargs...) + + problem = SemModelBlackBoxOptimProblem(model, optimizer) + if Method == :adaptive_de_rand_1_bin_with_gradient + # custom adaptive differential evolution with mutation that moves along the gradient + bbopt_factory = DefaultDiffEvoFactory(problem; kwargs...) + bbopt = bbsetup(bbopt_factory; MaxSteps, kwargs...) + else + bbopt = bbsetup(problem; Method, MaxSteps, kwargs...) + end + res = bboptimize(bbopt) + return SemFit(best_fitness(res), best_candidate(res), + nothing, model, res) +end From 77655a30e99a75a2bafa5cd9ddedc261847ff324 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 13:32:50 -0700 Subject: [PATCH 161/174] CommutationMatrix type replace comm_matrix helper functions with a CommutationMatrix and overloaded linalg ops --- src/StructuralEquationModels.jl | 4 + .../commutation_matrix.jl | 50 ++++++++ src/additional_functions/helper.jl | 108 ------------------ src/loss/ML/FIML.jl | 11 +- 4 files changed, 59 insertions(+), 114 deletions(-) create mode 100644 src/additional_functions/commutation_matrix.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 90e436cf4..37779acd8 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -14,6 +14,10 @@ const SEM = StructuralEquationModels # type hierarchy include("types.jl") include("objective_gradient_hessian.jl") + +# helper objects and functions +include("additional_functions/commutation_matrix.jl") + # fitted objects include("frontend/fit/SemFit.jl") # specification of models diff --git a/src/additional_functions/commutation_matrix.jl b/src/additional_functions/commutation_matrix.jl new file mode 100644 index 000000000..f7662fc38 --- /dev/null +++ b/src/additional_functions/commutation_matrix.jl @@ -0,0 +1,50 @@ +# transpose linear indices of the square n×n matrix +# i.e. +# 1 4 +# 2 5 => 1 2 3 +# 3 6 4 5 6 +transpose_linear_indices(n::Integer, m::Integer = n) = + repeat(1:n, inner = m) .+ repeat((0:(m-1))*n, outer = n) + +""" + CommutationMatrix(n::Integer) <: AbstractMatrix{Int} + +A *commutation matrix* *C* is a n²×n² matrix of 0s and 1s. +If *vec(A)* is a vectorized form of a n×n matrix *A*, +then ``C * vec(A) = vec(Aᵀ)``. +""" +struct CommutationMatrix <: AbstractMatrix{Int} + n::Int + n²::Int + transpose_inds::Vector{Int} # maps the linear indices of n×n matrix *B* to the indices of matrix *B'* + + CommutationMatrix(n::Integer) = + new(n, n^2, transpose_linear_indices(n)) +end + +Base.size(A::CommutationMatrix) = (A.n², A.n²) +Base.size(A::CommutationMatrix, dim::Integer) = + 1 <= dim <= 2 ? A.n² : throw(ArgumentError("invalid matrix dimension $dim")) +Base.length(A::CommutationMatrix) = A.n²^2 +Base.getindex(A::CommutationMatrix, i::Int, j::Int) = + j == A.transpose_inds[i] ? 1 : 0 + +function Base.:(*)(A::CommutationMatrix, B::AbstractMatrix) + size(A, 2) == size(B, 1) || throw(DimensionMismatch("A has $(size(A, 2)) columns, but B has $(size(B, 1)) rows")) + return B[A.transpose_inds, :] +end + +function Base.:(*)(A::CommutationMatrix, B::SparseMatrixCSC) + size(A, 2) == size(B, 1) || throw(DimensionMismatch("A has $(size(A, 2)) columns, but B has $(size(B, 1)) rows")) + return SparseMatrixCSC(size(B, 1), size(B, 2), + copy(B.colptr), A.transpose_inds[B.rowval], copy(B.nzval)) +end + +function LinearAlgebra.lmul!(A::CommutationMatrix, B::SparseMatrixCSC) + size(A, 2) == size(B, 1) || throw(DimensionMismatch("A has $(size(A, 2)) columns, but B has $(size(B, 1)) rows")) + + @inbounds for (i, rowind) in enumerate(B.rowval) + B.rowval[i] = A.transpose_inds[rowind] + end + return B +end diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index fc7a124f4..296eb905f 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -145,114 +145,6 @@ function elimination_matrix(nobs) return L end -function commutation_matrix(n; tosparse = false) - - M = zeros(n^2, n^2) - - for i = 1:n - for j = 1:n - M[i + n*(j - 1), j + n*(i - 1)] = 1.0 - end - end - - if tosparse M = sparse(M) end - - return M - -end - -function commutation_matrix_pre_square(A) - - n2 = size(A, 1) - n = Int(sqrt(n2)) - - ind = repeat(1:n, inner = n) - indadd = (0:(n-1))*n - for i in 1:n ind[((i-1)*n+1):i*n] .+= indadd end - - A_post = A[ind, :] - - return A_post - -end - -function commutation_matrix_pre_square_add!(B, A) # comuptes B + KₙA - - n2 = size(A, 1) - n = Int(sqrt(n2)) - - ind = repeat(1:n, inner = n) - indadd = (0:(n-1))*n - for i in 1:n ind[((i-1)*n+1):i*n] .+= indadd end - - @views @inbounds B .+= A[ind, :] - - return B - -end - -function get_commutation_lookup(n2::Int64) - - n = Int(sqrt(n2)) - ind = repeat(1:n, inner = n) - indadd = (0:(n-1))*n - for i in 1:n ind[((i-1)*n+1):i*n] .+= indadd end - - lookup = Dict{Int64, Int64}() - - for i in 1:n2 - j = findall(x -> (x == i), ind)[1] - push!(lookup, i => j) - end - - return lookup - -end - -function commutation_matrix_pre_square!(A::SparseMatrixCSC, lookup) # comuptes B + KₙA - - for (i, rowind) in enumerate(A.rowval) - A.rowval[i] = lookup[rowind] - end - -end - -function commutation_matrix_pre_square!(A::SparseMatrixCSC) # computes KₙA - lookup = get_commutation_lookup(size(A, 2)) - commutation_matrix_pre_square!(A, lookup) -end - -function commutation_matrix_pre_square(A::SparseMatrixCSC) - B = copy(A) - commutation_matrix_pre_square!(B) - return B -end - -function commutation_matrix_pre_square(A::SparseMatrixCSC, lookup) - B = copy(A) - commutation_matrix_pre_square!(B, lookup) - return B -end - - -function commutation_matrix_pre_square_add_mt!(B, A) # comuptes B + KₙA # 0 allocations but slower - - n2 = size(A, 1) - n = Int(sqrt(n2)) - - indadd = (0:(n-1))*n - - Threads.@threads for i = 1:n - for j = 1:n - row = i + indadd[j] - @views @inbounds B[row, :] .+= A[row, :] - end - end - - return B - -end - # returns the vector of non-unique values in the order of appearance # each non-unique values is reported once function nonunique(values::AbstractVector) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index cd9466e69..efd440037 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -91,7 +91,7 @@ struct SemFIML{T, W} <: SemLossFunction{ExactHessian} imp_inv::Matrix{T} # implied inverse - commutation_indices::Dict{Int, Int} + commutator::CommutationMatrix interaction::W end @@ -103,7 +103,7 @@ end function SemFIML(; observed::SemObservedMissing, specification, kwargs...) return SemFIML([SemFIMLPattern(pat) for pat in observed.patterns], zeros(n_man(observed), n_man(observed)), - get_commutation_lookup(nvars(specification)^2), nothing) + CommutationMatrix(nvars(specification)), nothing) end ############################################################################################ @@ -148,13 +148,12 @@ end function ∇F_fiml_outer!(G, JΣ, Jμ, fiml::SemFIML, imply, model) - Iₙ = sparse(1.0I, size(imply.A)...) P = kron(imply.F⨉I_A⁻¹, imply.F⨉I_A⁻¹) + Iₙ = sparse(1.0I, size(imply.A)...) Q = kron(imply.S*imply.I_A⁻¹', Iₙ) - #commutation_matrix_pre_square_add!(Q, Q) - Q2 = commutation_matrix_pre_square(Q, fiml.commutation_indices) + Q .+= fiml.commutator * Q - ∇Σ = P*(imply.∇S + (Q+Q2)*imply.∇A) + ∇Σ = P*(imply.∇S + Q*imply.∇A) ∇μ = imply.F⨉I_A⁻¹*imply.∇M + kron((imply.I_A⁻¹*imply.M)', imply.F⨉I_A⁻¹)*imply.∇A From 75138e8d4bf37769551cab3634bfe228c13917cf Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 13:21:02 -0700 Subject: [PATCH 162/174] simplify elimination_matrix() --- src/additional_functions/helper.jl | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 296eb905f..4381ce189 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -127,19 +127,17 @@ function duplication_matrix(nobs) return D end -function elimination_matrix(nobs) - nobs = Int(nobs) - n1 = Int(nobs*(nobs+1)*0.5) - n2 = Int(nobs^2) - L = zeros(n1, n2) - - for j in 1:nobs - for i in j:nobs - u = zeros(n1) - u[Int((j-1)*nobs + i-0.5*j*(j-1))] = 1 - T = zeros(nobs, nobs) - T[i, j] = 1 - L += u*transpose(vec(T)) +# (n(n+1)/2)×n² matrix to transform a +# vectorized form of a n×n symmetric matrix +# into vector of its lower triangular entries, +# opposite of duplication_matrix() +function elimination_matrix(n::Integer) + ntri = div(n*(n+1), 2) + L = zeros(ntri, n^2) + for j in 1:n + for i in j:n + tri_ix = (j-1)*n + i - div(j*(j-1), 2) + L[tri_ix, i + n*(j-1)] = 1 end end return L From 3a6f596ae0f65ebd6afe2d8e4af8a7bf7bdbfd51 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 00:43:28 -0700 Subject: [PATCH 163/174] simplify duplication_matrix() --- src/additional_functions/helper.jl | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/additional_functions/helper.jl b/src/additional_functions/helper.jl index 4381ce189..a7c0cd15e 100644 --- a/src/additional_functions/helper.jl +++ b/src/additional_functions/helper.jl @@ -108,22 +108,19 @@ function sparse_outer_mul!(C, A, B::Vector, ind) #computes A*S*B -> C, where ind end end -function duplication_matrix(nobs) - nobs = Int(nobs) - n1 = Int(nobs*(nobs+1)*0.5) - n2 = Int(nobs^2) - Dt = zeros(n1, n2) - - for j in 1:nobs - for i in j:nobs - u = zeros(n1) - u[Int((j-1)*nobs + i-0.5*j*(j-1))] = 1 - T = zeros(nobs, nobs) - T[j,i] = 1; T[i, j] = 1 - Dt += u*transpose(vec(T)) +# n²×(n(n+1)/2) matrix to transform a vector of lower +# triangular entries into a vectorized form of a n×n symmetric matrix, +# opposite of elimination_matrix() +function duplication_matrix(n::Integer) + ntri = div(n*(n+1), 2) + D = zeros(n^2, ntri) + for j in 1:n + for i in j:n + tri_ix = (j-1)*n + i - div(j*(j-1), 2) + D[j + n*(i-1), tri_ix] = 1 + D[i + n*(j-1), tri_ix] = 1 end end - D = transpose(Dt) return D end From 5c4ac3cd0d060cc9461b583c066f758011be12f8 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 00:43:54 -0700 Subject: [PATCH 164/174] FIML: optimize Jmu --- src/loss/ML/FIML.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/loss/ML/FIML.jl b/src/loss/ML/FIML.jl index efd440037..194394937 100644 --- a/src/loss/ML/FIML.jl +++ b/src/loss/ML/FIML.jl @@ -57,9 +57,10 @@ function gradient!(JΣ, Jμ, fiml::SemFIMLPattern, pat::SemObservedMissingPatter else JΣ_pat = Σ⁻¹ * (I - fiml.μ_diff * μ_diff⨉Σ⁻¹) end - Jμ_pat = ((2*n_obs(pat))*μ_diff⨉Σ⁻¹)' - vec(JΣ)[fiml.∇ind] .+= vec(JΣ_pat) - Jμ[pat.obs_mask] .+= Jμ_pat + @inbounds vec(JΣ)[fiml.∇ind] .+= vec(JΣ_pat) + + lmul!(2*n_obs(pat), μ_diff⨉Σ⁻¹) + @inbounds Jμ[pat.obs_mask] .+= μ_diff⨉Σ⁻¹' return nothing end From 09c93c8b498c343c2180b91b91591961f8584015 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 00:45:54 -0700 Subject: [PATCH 165/174] fix typo --- src/loss/WLS/WLS.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index 60f63c366..a705b6c89 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -22,7 +22,7 @@ Weighted least squares estimation. - `approximate_hessian::Bool`: should the hessian be swapped for an approximation - `wls_weight_matrix`: the weight matrix for weighted least squares. Defaults to GLS estimation (``0.5*(D^T*kron(S,S)*D)`` where D is the duplication matrix - and S is the inverse ob the observed covariance matrix) + and S is the inverse of the observed covariance matrix) - `wls_weight_matrix_mean`: the weight matrix for the mean part of weighted least squares. Defaults to GLS estimation (the inverse of the observed covariance matrix) From f281de1e62753fe6c125761969564b4a557b96e5 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 3 Apr 2024 00:46:20 -0700 Subject: [PATCH 166/174] SemWLS: dim checks --- src/loss/WLS/WLS.jl | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/loss/WLS/WLS.jl b/src/loss/WLS/WLS.jl index a705b6c89..32f57a25f 100644 --- a/src/loss/WLS/WLS.jl +++ b/src/loss/WLS/WLS.jl @@ -51,25 +51,35 @@ end SemWLS{HE}(args...) where {HE <: HessianEvaluation} = SemWLS{HE, map(typeof, args)...}(args...) -function SemWLS(;observed, wls_weight_matrix = nothing, wls_weight_matrix_mean = nothing, +function SemWLS(; observed, + wls_weight_matrix = nothing, + wls_weight_matrix_mean = nothing, approximate_hessian = false, meanstructure = false, kwargs...) - ind = CartesianIndices(obs_cov(observed)) - ind = filter(x -> (x[1] >= x[2]), ind) - s = obs_cov(observed)[ind] + n_obs = n_man(observed) + tril_ind = filter(x -> (x[1] >= x[2]), CartesianIndices(obs_cov(observed))) + s = obs_cov(observed)[tril_ind] # compute V here if isnothing(wls_weight_matrix) - D = duplication_matrix(n_man(observed)) + D = duplication_matrix(n_obs) S = inv(obs_cov(observed)) S = kron(S, S) wls_weight_matrix = 0.5*(D'*S*D) + else + size(wls_weight_matrix) == (length(tril_ind), length(tril_ind)) || + DimensionMismatch("wls_weight_matrix has to be of size $(length(tril_ind))×$(length(tril_ind))") end if meanstructure if isnothing(wls_weight_matrix_mean) wls_weight_matrix_mean = inv(obs_cov(observed)) + else + size(wls_weight_matrix_mean) == (n_obs, n_obs) || + DimensionMismatch("wls_weight_matrix_mean has to be of size $(n_obs)×$(n_obs)") end else + isnothing(wls_weight_matrix_mean) || + @warn "Ignoring wls_weight_matrix_mean since meanstructure is disabled" wls_weight_matrix_mean = nothing end HE = approximate_hessian ? ApproximateHessian : ExactHessian From 7cc2f585b0bc01d76b9f3b94bdb07b2eb4fad453 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 10 Apr 2024 15:39:49 -0700 Subject: [PATCH 167/174] EM: move code refs to docstring --- src/observed/EM.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 0c25f0ea2..11a98683c 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -2,15 +2,8 @@ ### Expectation Maximization Algorithm ############################################################################################ -# An EM Algorithm for MVN-distributed Data with missing values -# Adapted from supplementary Material to the book Machine Learning: A Probabilistic Perspective -# Copyright (2010) Kevin Murphy and Matt Dunham -# found at https://github.com/probml/pmtk3/blob/master/toolbox/BasicModels/gauss/sub/gaussMissingFitEm.m -# and at https://github.com/probml/pmtk3/blob/master/toolbox/Algorithms/optimization/emAlgo.m - # what about random restarts? -# outer function --------------------------------------------------------------------------- """ em_mvn(; observed::SemObservedMissing, @@ -21,6 +14,12 @@ Estimates the covariance matrix and mean vector of the normal distribution via expectation maximization for `observed`. Overwrites the statistics stored in `observed`. + +Uses the EM algorithm for MVN-distributed data with missing values +adapted from the supplementary material to the book *Machine Learning: A Probabilistic Perspective*, +copyright (2010) Kevin Murphy and Matt Dunham: see +[*gaussMissingFitEm.m*](https://github.com/probml/pmtk3/blob/master/toolbox/BasicModels/gauss/sub/gaussMissingFitEm.m) and +[*emAlgo.m*](https://github.com/probml/pmtk3/blob/master/toolbox/Algorithms/optimization/emAlgo.m) scripts. """ function em_mvn( observed::SemObservedMissing; From 1b30da44b6def6ae6f8f2f618edb301fe922a857 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 10 Apr 2024 15:41:29 -0700 Subject: [PATCH 168/174] EM MVN: decouple from SemObsMissing so EM MVN could be done when SemObsMissing is constructed --- src/StructuralEquationModels.jl | 3 +- .../start_val/start_fabin3.jl | 21 +- src/frontend/fit/fitmeasures/minus2ll.jl | 5 +- src/observed/EM.jl | 180 +++++++++--------- src/observed/missing.jl | 71 +------ src/observed/missing_pattern.jl | 43 +++++ 6 files changed, 144 insertions(+), 179 deletions(-) create mode 100644 src/observed/missing_pattern.jl diff --git a/src/StructuralEquationModels.jl b/src/StructuralEquationModels.jl index 37779acd8..68faa2fe9 100644 --- a/src/StructuralEquationModels.jl +++ b/src/StructuralEquationModels.jl @@ -33,8 +33,9 @@ include("frontend/pretty_printing.jl") # observed include("observed/data.jl") include("observed/covariance.jl") -include("observed/missing.jl") +include("observed/missing_pattern.jl") include("observed/EM.jl") +include("observed/missing.jl") # constructor include("frontend/specification/Sem.jl") include("frontend/specification/documentation.jl") diff --git a/src/additional_functions/start_val/start_fabin3.jl b/src/additional_functions/start_val/start_fabin3.jl index c29e17727..026897363 100644 --- a/src/additional_functions/start_val/start_fabin3.jl +++ b/src/additional_functions/start_val/start_fabin3.jl @@ -19,7 +19,7 @@ function start_fabin3( end function start_fabin3( - observed, + observed::SemObserved, imply, optimizer, args...; @@ -30,25 +30,6 @@ function start_fabin3( obs_mean(observed)) end -# SemObservedMissing -function start_fabin3( - observed::SemObservedMissing, - imply, - optimizer, - args...; - kwargs...) - - if !observed.em_model.fitted - em_mvn(observed; kwargs...) - end - - return start_fabin3( - imply.ram_matrices, - observed.em_model.Σ, - observed.em_model.μ) -end - - function start_fabin3(ram_matrices::RAMMatrices, Σ::AbstractMatrix, μ::Union{AbstractVector, Nothing}) diff --git a/src/frontend/fit/fitmeasures/minus2ll.jl b/src/frontend/fit/fitmeasures/minus2ll.jl index 6d6b61997..c694aa12d 100644 --- a/src/frontend/fit/fitmeasures/minus2ll.jl +++ b/src/frontend/fit/fitmeasures/minus2ll.jl @@ -41,10 +41,7 @@ end # compute likelihood for missing data - H1 ------------------------------------------------- # -2ll = ∑ log(2π)*(nᵢ + mᵢ) + ln(Σᵢ) + (mᵢ - μᵢ)ᵀ Σᵢ⁻¹ (mᵢ - μᵢ)) + tr(SᵢΣᵢ) function minus2ll(observed::SemObservedMissing) - observed.em_model.fitted || em_mvn(observed) - - μ = observed.em_model.μ - Σ = observed.em_model.Σ + Σ, μ = obs_cov(observed), obs_mean(observed) F = 0.0 for pat in observed.patterns diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 11a98683c..bec466500 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -5,15 +5,17 @@ # what about random restarts? """ - em_mvn(; - observed::SemObservedMissing, - start_em = start_em_observed, - max_iter_em = 100, - rtol_em = 1e-4, - kwargs...) + em_mvn(patterns::AbstractVector{SemObservedMissingPattern}; + start_em = start_em_observed, + max_iter_em = 100, + rtol_em = 1e-4, + kwargs...) -Estimates the covariance matrix and mean vector of the normal distribution via expectation maximization for `observed`. -Overwrites the statistics stored in `observed`. +Estimates the covariance matrix and mean vector of the +multivariate normal distribution (MVN) +via expectation maximization (EM) for `observed`. + +Returns the tuple of the EM covariance matrix and the EM mean vector. Uses the EM algorithm for MVN-distributed data with missing values adapted from the supplementary material to the book *Machine Learning: A Probabilistic Perspective*, @@ -22,23 +24,21 @@ copyright (2010) Kevin Murphy and Matt Dunham: see [*emAlgo.m*](https://github.com/probml/pmtk3/blob/master/toolbox/Algorithms/optimization/emAlgo.m) scripts. """ function em_mvn( - observed::SemObservedMissing; + patterns::AbstractVector{<:SemObservedMissingPattern}; start_em = start_em_observed, - max_iter_em = 100, - rtol_em = 1e-4, + max_iter_em::Integer = 100, + rtol_em::Number = 1e-4, kwargs...) - n_man = SEM.n_man(observed) - - # preallocate stuff? - 𝔼x_pre = zeros(n_man) - 𝔼xxᵀ_pre = zeros(n_man, n_man) + n_man = SEM.n_man(patterns[1]) ### precompute for full cases - fullpat = observed.patterns[1] - if nmissed_vars(fullpat) == 0 - sum!(reshape(𝔼x_pre, 1, n_man), fullpat.data) - mul!(𝔼xxᵀ_pre, fullpat.data', fullpat.data) + 𝔼x_full = zeros(n_man) + 𝔼xxᵀ_full = zeros(n_man, n_man) + if nmissed_vars(patterns[1]) == 0 + fullpat = patterns[1] + sum!(reshape(𝔼x_full, 1, n_man), fullpat.data) + mul!(𝔼xxᵀ_full, fullpat.data', fullpat.data) else @warn "No full cases pattern found" end @@ -47,57 +47,56 @@ function em_mvn( # estepFn = (em_model, data) -> estep(em_model, data, EXsum, EXXsum, ismissing, missingRows, n_obs) # initialize - em_model = start_em(observed; kwargs...) - em_model_prev = EmMVNModel(zeros(n_man, n_man), zeros(n_man), false) - iter = 1 - done = false - 𝔼x = zeros(n_man) - 𝔼xxᵀ = zeros(n_man, n_man) - - while !done - - step!(em_model, observed, 𝔼x, 𝔼xxᵀ, 𝔼x_pre, 𝔼xxᵀ_pre) - - if iter > max_iter_em - done = true - @warn "EM Algorithm for MVN missing data did not converge. Likelihood for FIML is not interpretable. - Maybe try passing different starting values via 'start_em = ...' " - elseif iter > 1 - # done = isapprox(ll, ll_prev; rtol = rtol) - done = isapprox(em_model_prev.μ, em_model.μ; rtol = rtol_em) && - isapprox(em_model_prev.Σ, em_model.Σ; rtol = rtol_em) + Σ₀, μ = start_em(patterns; kwargs...) + Σ = convert(Matrix, Σ₀) + @assert all(isfinite, Σ) all(isfinite, μ) + Σ_prev, μ_prev = copy(Σ), copy(μ) + + iter = 0 + converged = false + while !converged && (iter < max_iter_em) + em_step!(Σ, μ, Σ_prev, μ_prev, patterns, 𝔼x_full, 𝔼xxᵀ_full) + + if iter > 0 + Δμ = norm(μ - μ_prev) + ΔΣ = norm(Σ - Σ_prev) + Δμ_rel = Δμ / max(norm(μ_prev), norm(μ)) + ΔΣ_rel = ΔΣ / max(norm(Σ_prev), norm(Σ)) + #@info "Iteration #$iter: ΔΣ=$(ΔΣ) ΔΣ/Σ=$(ΔΣ_rel) Δμ=$(Δμ) Δμ/μ=$(Δμ_rel)" + # converged = isapprox(ll, ll_prev; rtol = rtol) + converged = ΔΣ_rel <= rtol_em && Δμ_rel <= rtol_em + end + if !converged + Σ, Σ_prev = Σ_prev, Σ + μ, μ_prev = μ_prev, μ end - - # print("$iter \n") iter += 1 - copyto!(em_model_prev.μ, em_model.μ) - copyto!(em_model_prev.Σ, em_model.Σ) - + #@info "$iter\n" end - # update EM Mode in observed - observed.em_model.Σ .= em_model.Σ - observed.em_model.μ .= em_model.μ - observed.em_model.fitted = true - - return nothing + if !converged + @warn "EM Algorithm for MVN missing data did not converge in $iter iterations.\n" * + "Likelihood for FIML is not interpretable.\n" * + "Maybe try passing different starting values via 'start_em = ...' " + else + @info "EM for MVN missing data converged in $iter iterations" + end + return Σ, μ end # E and M steps ----------------------------------------------------------------------------- -# update em_model -function step!(em_model::EmMVNModel, observed::SemObserved, - 𝔼x, 𝔼xxᵀ, 𝔼x_pre, 𝔼xxᵀ_pre) +function em_step!(Σ::AbstractMatrix, μ::AbstractVector, + Σ₀::AbstractMatrix, μ₀::AbstractVector, + patterns::AbstractVector{<:SemObservedMissingPattern}, + 𝔼x_full, 𝔼xxᵀ_full) # E step, update 𝔼x and 𝔼xxᵀ - fill!(𝔼x, 0) - fill!(𝔼xxᵀ, 0) - - μ = em_model.μ - Σ = em_model.Σ + copy!(μ, 𝔼x_full) + copy!(Σ, 𝔼xxᵀ_full) # Compute the expected sufficient statistics - for pat in observed.patterns + for pat in patterns (nmissed_vars(pat) == 0) && continue # skip full cases # observed and unobserved vars @@ -105,17 +104,17 @@ function step!(em_model::EmMVNModel, observed::SemObserved, o = pat.obs_mask # precompute for pattern - Σoo_chol = cholesky(Symmetric(Σ[o, o])) - Σuo = Σ[u, o] - μu = μ[u] - μo = μ[o] + Σoo_chol = cholesky(Symmetric(Σ₀[o, o])) + Σuo = Σ₀[u, o] + μu = μ₀[u] + μo = μ₀[o] 𝔼xu = fill!(similar(μu), 0) 𝔼xo = fill!(similar(μo), 0) 𝔼xᵢu = similar(μu) 𝔼xxᵀuo = fill!(similar(Σuo), 0) - 𝔼xxᵀuu = n_obs(pat) * (Σ[u, u] - Σuo * (Σoo_chol \ Σuo')) + 𝔼xxᵀuu = n_obs(pat) * (Σ₀[u, u] - Σuo * (Σoo_chol \ Σuo')) # loop trough data @inbounds for rowdata in eachrow(pat.data) @@ -127,24 +126,21 @@ function step!(em_model::EmMVNModel, observed::SemObserved, 𝔼xo .+= rowdata end - 𝔼xxᵀ[o,o] .+= pat.data' * pat.data - 𝔼xxᵀ[u,o] .+= 𝔼xxᵀuo - 𝔼xxᵀ[o,u] .+= 𝔼xxᵀuo' - 𝔼xxᵀ[u,u] .+= 𝔼xxᵀuu + Σ[o,o] .+= pat.data' * pat.data + Σ[u,o] .+= 𝔼xxᵀuo + Σ[o,u] .+= 𝔼xxᵀuo' + Σ[u,u] .+= 𝔼xxᵀuu - 𝔼x[o] .+= 𝔼xo - 𝔼x[u] .+= 𝔼xu + μ[o] .+= 𝔼xo + μ[u] .+= 𝔼xu end - 𝔼x .+= 𝔼x_pre - 𝔼xxᵀ .+= 𝔼xxᵀ_pre - # M step, update em_model - em_model.μ .= 𝔼x ./ n_obs(observed) - em_model.Σ .= 𝔼xxᵀ ./ n_obs(observed) - mul!(em_model.Σ, em_model.μ, em_model.μ', -1, 1) + k = inv(sum(n_obs, patterns)) + lmul!(k, Σ) + lmul!(k, μ) + mul!(Σ, μ, μ', -1, 1) - #Σ = em_model.Σ # ridge Σ # while !isposdef(Σ) # Σ += 0.5I @@ -153,28 +149,28 @@ function step!(em_model::EmMVNModel, observed::SemObserved, # diagonalization #if !isposdef(Σ) # print("Matrix not positive definite") - # em_model.Σ .= 0 - # em_model.Σ[diagind(em_model.Σ)] .= diag(Σ) + # Σ .= 0 + # Σ[diagind(em_model.Σ)] .= diag(Σ) #else - # em_model.Σ = Σ + # Σ = Σ #end - return em_model + return Σ, μ end # generate starting values ----------------------------------------------------------------- # use μ and Σ of full cases -function start_em_observed(observed::SemObservedMissing; kwargs...) +function start_em_observed(patterns::AbstractVector{<:SemObservedMissingPattern}; kwargs...) - fullpat = observed.patterns[1] + fullpat = patterns[1] if (nmissed_vars(fullpat) == 0) && (n_obs(fullpat) > 1) μ = copy(fullpat.obs_mean) - Σ = copy(fullpat.obs_cov) + Σ = copy(parent(fullpat.obs_cov)) if !isposdef(Σ) Σ = Diagonal(Σ) end - return EmMVNModel(convert(Matrix, Σ), μ, false) + return Σ, μ else return start_em_simple(observed, kwargs...) end @@ -182,15 +178,17 @@ function start_em_observed(observed::SemObservedMissing; kwargs...) end # use μ = O and Σ = I -function start_em_simple(observed::SemObservedMissing; kwargs...) - μ = zeros(n_man(observed)) - Σ = rand(n_man(observed), n_man(observed)) +function start_em_simple(patterns::AbstractVector{<:SemObservedMissingPattern}; kwargs...) + nvars = n_man(first(patterns)) + μ = zeros(nvars) + Σ = rand(nvars, nvars) Σ = Σ*Σ' # Σ = Matrix(1.0I, n_man, n_man) - return EmMVNModel(Σ, μ, false) + return Σ, μ end # set to passed values -function start_em_set(observed::SemObservedMissing; model_em, kwargs...) - return em_model +function start_em_set(patterns::AbstractVector{<:SemObservedMissingPattern}; + obs_cov::AbstractMatrix, obs_mean::AbstractVector, kwargs...) + return copy(obs_cov), copy(obs_mean) end \ No newline at end of file diff --git a/src/observed/missing.jl b/src/observed/missing.jl index 8c66ffda8..a1f622f4e 100644 --- a/src/observed/missing.jl +++ b/src/observed/missing.jl @@ -2,57 +2,6 @@ ### Types ############################################################################################ -# Type to store Expectation Maximization result -------------------------------------------- -mutable struct EmMVNModel{A, b, B} - Σ::A - μ::b - fitted::B -end - -# data associated with the specific pattern of missed variable -struct SemObservedMissingPattern{T,S} - obs_mask::BitVector # observed vars mask - miss_mask::BitVector # missing vars mask - nobserved::Int - nmissed::Int - rows::Vector{Int} # rows in original data - data::Matrix{T} # non-missing submatrix of data - - obs_mean::Vector{S} # means of observed vars - obs_cov::Symmetric{S, Matrix{S}} # covariance of observed vars -end - -function SemObservedMissingPattern( - obs_mask::BitVector, - rows::AbstractVector{<:Integer}, - data::AbstractMatrix -) - T = nonmissingtype(eltype(data)) - - pat_data = convert(Matrix{T}, view(data, rows, obs_mask)) - if size(pat_data, 1) > 1 - pat_mean, pat_cov = mean_and_cov(pat_data, 1, corrected=false) - @assert size(pat_cov) == (size(pat_data, 2), size(pat_data, 2)) - else - pat_mean = reshape(pat_data[1, :], 1, :) - pat_cov = fill(zero(T), 1, 1) - end - - miss_mask = .!obs_mask - - return SemObservedMissingPattern{T, eltype(pat_mean)}( - obs_mask, miss_mask, - sum(obs_mask), sum(miss_mask), - rows, pat_data, - dropdims(pat_mean, dims=1), Symmetric(pat_cov)) -end - -n_man(pat::SemObservedMissingPattern) = length(pat.obs_mask) -nobserved_vars(pat::SemObservedMissingPattern) = pat.nobserved -nmissed_vars(pat::SemObservedMissingPattern) = pat.nmissed - -n_obs(pat::SemObservedMissingPattern) = length(pat.rows) - """ For observed data with missing values. @@ -91,13 +40,15 @@ use this if you are sure your observed data is in the right format. struct SemObservedMissing{ A <: AbstractMatrix, P <: SemObservedMissingPattern, - S <: EmMVNModel + T <: Real } <: SemObserved data::A n_man::Int n_obs::Int patterns::Vector{P} - em_model::S + + obs_cov::Matrix{T} + obs_mean::Vector{T} end ############################################################################################ @@ -164,19 +115,13 @@ function SemObservedMissing(; for (pat, rows) in pairs(pattern_to_rows)] sort!(patterns, by=nmissed_vars) - # allocate EM model (but don't fit) - em_model = EmMVNModel(zeros(n_man, n_man), zeros(n_man), false) + em_cov, em_mean = em_mvn(patterns; kwargs...) - return SemObservedMissing(data, n_man, n_obs, patterns, em_model) + return SemObservedMissing(data, n_man, n_obs, patterns, em_cov, em_mean) end -############################################################################################ -### Recommended methods -############################################################################################ - n_obs(observed::SemObservedMissing) = observed.n_obs n_man(observed::SemObservedMissing) = observed.n_man -############################################################################################ -### Additional methods -############################################################################################ +obs_cov(observed::SemObservedMissing) = observed.obs_cov +obs_mean(observed::SemObservedMissing) = observed.obs_mean diff --git a/src/observed/missing_pattern.jl b/src/observed/missing_pattern.jl new file mode 100644 index 000000000..3e059e806 --- /dev/null +++ b/src/observed/missing_pattern.jl @@ -0,0 +1,43 @@ +# data associated with the specific pattern of missing manifested variables +struct SemObservedMissingPattern{T,S} + obs_mask::BitVector # observed vars mask + miss_mask::BitVector # missing vars mask + nobserved::Int + nmissed::Int + rows::Vector{Int} # rows in original data + data::Matrix{T} # non-missing submatrix of data + + obs_mean::Vector{S} # means of observed vars + obs_cov::Symmetric{S, Matrix{S}} # covariance of observed vars +end + +function SemObservedMissingPattern( + obs_mask::BitVector, + rows::AbstractVector{<:Integer}, + data::AbstractMatrix +) + T = nonmissingtype(eltype(data)) + + pat_data = convert(Matrix{T}, view(data, rows, obs_mask)) + if size(pat_data, 1) > 1 + pat_mean, pat_cov = mean_and_cov(pat_data, 1, corrected=false) + @assert size(pat_cov) == (size(pat_data, 2), size(pat_data, 2)) + else + pat_mean = reshape(pat_data[1, :], 1, :) + pat_cov = fill(zero(T), 1, 1) + end + + miss_mask = .!obs_mask + + return SemObservedMissingPattern{T, eltype(pat_mean)}( + obs_mask, miss_mask, + sum(obs_mask), sum(miss_mask), + rows, pat_data, + dropdims(pat_mean, dims=1), Symmetric(pat_cov)) +end + +n_man(pat::SemObservedMissingPattern) = length(pat.obs_mask) +n_obs(pat::SemObservedMissingPattern) = length(pat.rows) + +nobserved_vars(pat::SemObservedMissingPattern) = pat.nobserved +nmissed_vars(pat::SemObservedMissingPattern) = pat.nmissed From 0eb269099bd0f769b864938e85d6047899ad9fad Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 15:52:01 -0700 Subject: [PATCH 169/174] fixup semoptimizer in ext tests --- test/examples/political_democracy/by_parts.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index 7ac2d9c75..d42d533d7 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -130,10 +130,11 @@ end ### test hessians ############################################################################################ -if semoptimizer == SemOptimizerOptim +if opt_engine == :Optim using Optim, LineSearches - optimizer_obj = SemOptimizerOptim( + optimizer_obj = SemOptimizer( + engine = opt_engine, algorithm = Newton( ;linesearch = BackTracking(order=3), alphaguess = InitialHagerZhang() @@ -195,7 +196,7 @@ loss_ml = SemLoss(ml) loss_wls = SemLoss(wls) # optimizer ------------------------------------------------------------------------------------- -optimizer_obj = semoptimizer() +optimizer_obj = SemOptimizer(engine = opt_engine) # models ----------------------------------------------------------------------------------- model_ml = Sem(observed, imply_ram, loss_ml, optimizer_obj) From 090ef5ea4bb1020fb1504e02b46f690b94657b7f Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Sun, 14 Apr 2024 16:22:46 -0700 Subject: [PATCH 170/174] test/fiml: set EM MVN rtol=1e-10 to make tests pass --- test/examples/political_democracy/by_parts.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/examples/political_democracy/by_parts.jl b/test/examples/political_democracy/by_parts.jl index d42d533d7..9a5b98967 100644 --- a/test/examples/political_democracy/by_parts.jl +++ b/test/examples/political_democracy/by_parts.jl @@ -273,7 +273,7 @@ end ### fiml ############################################################################################ -observed = SemObservedMissing(specification = spec_mean, data = dat_missing) +observed = SemObservedMissing(specification = spec_mean, data = dat_missing, rtol_em = 1e-10) fiml = SemFIML(observed = observed, specification = spec_mean) From 1eb987aa9c360b8c45e27054e9628486081c5867 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 17 Apr 2024 01:05:58 -0700 Subject: [PATCH 171/174] MissingPattern: transpose data for faster EM MVN --- src/observed/EM.jl | 25 +++++++++++++------------ src/observed/missing_pattern.jl | 4 ++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index bec466500..3eb1835af 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -32,19 +32,20 @@ function em_mvn( n_man = SEM.n_man(patterns[1]) - ### precompute for full cases + # precompute for full cases 𝔼x_full = zeros(n_man) 𝔼xxᵀ_full = zeros(n_man, n_man) - if nmissed_vars(patterns[1]) == 0 - fullpat = patterns[1] - sum!(reshape(𝔼x_full, 1, n_man), fullpat.data) - mul!(𝔼xxᵀ_full, fullpat.data', fullpat.data) - else - @warn "No full cases pattern found" + nobs_full = 0 + for pat in patterns + if nmissed_vars(pat) == 0 + 𝔼x_full .+= sum(pat.data, dims=2) + mul!(𝔼xxᵀ_full, pat.data, pat.data', 1, 1) + nobs_full += n_obs(pat) + end + end + if nobs_full == 0 + @warn "No full cases in data" end - - # ess = 𝔼x, 𝔼xxᵀ, ismissing, missingRows, n_obs - # estepFn = (em_model, data) -> estep(em_model, data, EXsum, EXXsum, ismissing, missingRows, n_obs) # initialize Σ₀, μ = start_em(patterns; kwargs...) @@ -116,8 +117,8 @@ function em_step!(Σ::AbstractMatrix, μ::AbstractVector, 𝔼xxᵀuo = fill!(similar(Σuo), 0) 𝔼xxᵀuu = n_obs(pat) * (Σ₀[u, u] - Σuo * (Σoo_chol \ Σuo')) - # loop trough data - @inbounds for rowdata in eachrow(pat.data) + # loop through observations + @inbounds for rowdata in eachcol(pat.data) mul!(𝔼xᵢu, Σuo, Σoo_chol \ (rowdata-μo)) 𝔼xᵢu .+= μu mul!(𝔼xxᵀuu, 𝔼xᵢu, 𝔼xᵢu', 1, 1) diff --git a/src/observed/missing_pattern.jl b/src/observed/missing_pattern.jl index 3e059e806..03c6e7be2 100644 --- a/src/observed/missing_pattern.jl +++ b/src/observed/missing_pattern.jl @@ -5,7 +5,7 @@ struct SemObservedMissingPattern{T,S} nobserved::Int nmissed::Int rows::Vector{Int} # rows in original data - data::Matrix{T} # non-missing submatrix of data + data::Matrix{T} # non-missing submatrix of data (vars × observations) obs_mean::Vector{S} # means of observed vars obs_cov::Symmetric{S, Matrix{S}} # covariance of observed vars @@ -32,7 +32,7 @@ function SemObservedMissingPattern( return SemObservedMissingPattern{T, eltype(pat_mean)}( obs_mask, miss_mask, sum(obs_mask), sum(miss_mask), - rows, pat_data, + rows, permutedims(pat_data), dropdims(pat_mean, dims=1), Symmetric(pat_cov)) end From fd71ebd99068aa9279ecf24d81bea154a00412c3 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 17 Apr 2024 01:06:55 -0700 Subject: [PATCH 172/174] EM MVN: report rel_error if not converged --- src/observed/EM.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 3eb1835af..8d35b901d 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -55,6 +55,8 @@ function em_mvn( iter = 0 converged = false + Δμ_rel = NaN + ΔΣ_rel = NaN while !converged && (iter < max_iter_em) em_step!(Σ, μ, Σ_prev, μ_prev, patterns, 𝔼x_full, 𝔼xxᵀ_full) @@ -76,7 +78,7 @@ function em_mvn( end if !converged - @warn "EM Algorithm for MVN missing data did not converge in $iter iterations.\n" * + @warn "EM Algorithm for MVN missing data did not converge in $iter iterations (ΔΣ/Σ=$(ΔΣ_rel) Δμ/μ=$(Δμ_rel)).\n" * "Likelihood for FIML is not interpretable.\n" * "Maybe try passing different starting values via 'start_em = ...' " else From b93f1283b6594842bfcf7a79d2c4b18660e44e6b Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 17 Apr 2024 01:21:21 -0700 Subject: [PATCH 173/174] EM: max_nobs_em opt to limit obs used --- src/observed/EM.jl | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 8d35b901d..2155ed3b9 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -28,6 +28,7 @@ function em_mvn( start_em = start_em_observed, max_iter_em::Integer = 100, rtol_em::Number = 1e-4, + max_nobs_em::Union{Integer, Nothing} = nothing, kwargs...) n_man = SEM.n_man(patterns[1]) @@ -58,7 +59,8 @@ function em_mvn( Δμ_rel = NaN ΔΣ_rel = NaN while !converged && (iter < max_iter_em) - em_step!(Σ, μ, Σ_prev, μ_prev, patterns, 𝔼x_full, 𝔼xxᵀ_full) + em_step!(Σ, μ, Σ_prev, μ_prev, patterns, + 𝔼xxᵀ_full, 𝔼x_full, nobs_full; max_nobs_em) if iter > 0 Δμ = norm(μ - μ_prev) @@ -93,14 +95,17 @@ end function em_step!(Σ::AbstractMatrix, μ::AbstractVector, Σ₀::AbstractMatrix, μ₀::AbstractVector, patterns::AbstractVector{<:SemObservedMissingPattern}, - 𝔼x_full, 𝔼xxᵀ_full) + 𝔼xxᵀ_full::AbstractMatrix, 𝔼x_full::AbstractVector, nobs_full::Integer; + max_nobs_em::Union{Integer, Nothing} = nothing +) # E step, update 𝔼x and 𝔼xxᵀ copy!(μ, 𝔼x_full) copy!(Σ, 𝔼xxᵀ_full) + nobs_used = nobs_full # Compute the expected sufficient statistics for pat in patterns - (nmissed_vars(pat) == 0) && continue # skip full cases + (nmissed_vars(pat) == 0) && continue # full cases already accounted for # observed and unobserved vars u = pat.miss_mask @@ -112,6 +117,12 @@ function em_step!(Σ::AbstractMatrix, μ::AbstractVector, μu = μ₀[u] μo = μ₀[o] + # get pattern observations + nobs = !isnothing(max_nobs_em) ? min(max_nobs_em, n_obs(pat)) : n_obs(pat) + pat_data = nobs < n_obs(pat) ? + view(pat.data, :, sort!(sample(1:n_obs(pat), nobs, replace = false))) : + pat.data + 𝔼xu = fill!(similar(μu), 0) 𝔼xo = fill!(similar(μo), 0) 𝔼xᵢu = similar(μu) @@ -120,28 +131,29 @@ function em_step!(Σ::AbstractMatrix, μ::AbstractVector, 𝔼xxᵀuu = n_obs(pat) * (Σ₀[u, u] - Σuo * (Σoo_chol \ Σuo')) # loop through observations - @inbounds for rowdata in eachcol(pat.data) - mul!(𝔼xᵢu, Σuo, Σoo_chol \ (rowdata-μo)) + @inbounds for obsdata in eachcol(pat_data) + mul!(𝔼xᵢu, Σuo, Σoo_chol \ (obsdata-μo)) 𝔼xᵢu .+= μu mul!(𝔼xxᵀuu, 𝔼xᵢu, 𝔼xᵢu', 1, 1) - mul!(𝔼xxᵀuo, 𝔼xᵢu, rowdata', 1, 1) + mul!(𝔼xxᵀuo, 𝔼xᵢu, obsdata', 1, 1) 𝔼xu .+= 𝔼xᵢu - 𝔼xo .+= rowdata + 𝔼xo .+= obsdata end - Σ[o,o] .+= pat.data' * pat.data + Σ[o,o] .+= pat_data * pat_data' Σ[u,o] .+= 𝔼xxᵀuo Σ[o,u] .+= 𝔼xxᵀuo' Σ[u,u] .+= 𝔼xxᵀuu μ[o] .+= 𝔼xo μ[u] .+= 𝔼xu + + nobs_used += nobs end # M step, update em_model - k = inv(sum(n_obs, patterns)) - lmul!(k, Σ) - lmul!(k, μ) + lmul!(1/nobs_used, Σ) + lmul!(1/nobs_used, μ) mul!(Σ, μ, μ', -1, 1) # ridge Σ From 52c1a8bbdcbcfb5c49a0464d777b470b3b6288d3 Mon Sep 17 00:00:00 2001 From: Alexey Stukalov Date: Wed, 17 Apr 2024 01:23:43 -0700 Subject: [PATCH 174/174] EM: optimize mean handling --- src/observed/EM.jl | 74 +++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/src/observed/EM.jl b/src/observed/EM.jl index 2155ed3b9..1bc9cb348 100644 --- a/src/observed/EM.jl +++ b/src/observed/EM.jl @@ -17,7 +17,7 @@ via expectation maximization (EM) for `observed`. Returns the tuple of the EM covariance matrix and the EM mean vector. -Uses the EM algorithm for MVN-distributed data with missing values +Based on the EM algorithm for MVN-distributed data with missing values adapted from the supplementary material to the book *Machine Learning: A Probabilistic Perspective*, copyright (2010) Kevin Murphy and Matt Dunham: see [*gaussMissingFitEm.m*](https://github.com/probml/pmtk3/blob/master/toolbox/BasicModels/gauss/sub/gaussMissingFitEm.m) and @@ -102,6 +102,8 @@ function em_step!(Σ::AbstractMatrix, μ::AbstractVector, copy!(μ, 𝔼x_full) copy!(Σ, 𝔼xxᵀ_full) nobs_used = nobs_full + mul!(Σ, μ₀, μ₀', -nobs_used, 1) + axpy!(-nobs_used, μ₀, μ) # Compute the expected sufficient statistics for pat in patterns @@ -111,42 +113,55 @@ function em_step!(Σ::AbstractMatrix, μ::AbstractVector, u = pat.miss_mask o = pat.obs_mask - # precompute for pattern - Σoo_chol = cholesky(Symmetric(Σ₀[o, o])) - Σuo = Σ₀[u, o] - μu = μ₀[u] - μo = μ₀[o] + # compute cholesky to speed-up ldiv!() + Σ₀oo_chol = cholesky(Symmetric(Σ₀[o, o])) + Σ₀uo = Σ₀[u, o] + μ₀u = μ₀[u] + μ₀o = μ₀[o] # get pattern observations nobs = !isnothing(max_nobs_em) ? min(max_nobs_em, n_obs(pat)) : n_obs(pat) - pat_data = nobs < n_obs(pat) ? - view(pat.data, :, sort!(sample(1:n_obs(pat), nobs, replace = false))) : - pat.data + zo = nobs < n_obs(pat) ? + pat.data[:, sort!(sample(1:n_obs(pat), nobs, replace = false))] : + copy(pat.data) + zo .-= μ₀o # subtract current mean from observations - 𝔼xu = fill!(similar(μu), 0) - 𝔼xo = fill!(similar(μo), 0) - 𝔼xᵢu = similar(μu) + 𝔼zo = sum(zo, dims = 2) + 𝔼zu = fill!(similar(μ₀u), 0) - 𝔼xxᵀuo = fill!(similar(Σuo), 0) - 𝔼xxᵀuu = n_obs(pat) * (Σ₀[u, u] - Σuo * (Σoo_chol \ Σuo')) + 𝔼zzᵀuo = fill!(similar(Σ₀uo), 0) + 𝔼zzᵀuu = nobs * Σ₀[u, u] + mul!(𝔼zzᵀuu, Σ₀uo, Σ₀oo_chol \ Σ₀uo', -nobs, 1) # loop through observations - @inbounds for obsdata in eachcol(pat_data) - mul!(𝔼xᵢu, Σuo, Σoo_chol \ (obsdata-μo)) - 𝔼xᵢu .+= μu - mul!(𝔼xxᵀuu, 𝔼xᵢu, 𝔼xᵢu', 1, 1) - mul!(𝔼xxᵀuo, 𝔼xᵢu, obsdata', 1, 1) - 𝔼xu .+= 𝔼xᵢu - 𝔼xo .+= obsdata + yᵢo = similar(μ₀o) + 𝔼zᵢu = similar(μ₀u) + @inbounds for zᵢo in eachcol(zo) + ldiv!(yᵢo, Σ₀oo_chol, zᵢo) + mul!(𝔼zᵢu, Σ₀uo, yᵢo) + mul!(𝔼zzᵀuu, 𝔼zᵢu, 𝔼zᵢu', 1, 1) + mul!(𝔼zzᵀuo, 𝔼zᵢu, zᵢo', 1, 1) + 𝔼zu .+= 𝔼zᵢu end + # correct 𝔼zzᵀ by adding back μ₀×𝔼z' + 𝔼z'×μ₀ + mul!(𝔼zzᵀuo, μ₀u, 𝔼zo', 1, 1) + mul!(𝔼zzᵀuo, 𝔼zu, μ₀o', 1, 1) - Σ[o,o] .+= pat_data * pat_data' - Σ[u,o] .+= 𝔼xxᵀuo - Σ[o,u] .+= 𝔼xxᵀuo' - Σ[u,u] .+= 𝔼xxᵀuu + mul!(𝔼zzᵀuu, μ₀u, 𝔼zu', 1, 1) + mul!(𝔼zzᵀuu, 𝔼zu, μ₀u', 1, 1) - μ[o] .+= 𝔼xo - μ[u] .+= 𝔼xu + 𝔼zzᵀoo = zo * zo' + mul!(𝔼zzᵀoo, μ₀o, 𝔼zo', 1, 1) + mul!(𝔼zzᵀoo, 𝔼zo, μ₀o', 1, 1) + + # update Σ and μ + Σ[o,o] .+= 𝔼zzᵀoo + Σ[u,o] .+= 𝔼zzᵀuo + Σ[o,u] .+= 𝔼zzᵀuo' + Σ[u,u] .+= 𝔼zzᵀuu + + μ[o] .+= 𝔼zo + μ[u] .+= 𝔼zu nobs_used += nobs end @@ -154,7 +169,12 @@ function em_step!(Σ::AbstractMatrix, μ::AbstractVector, # M step, update em_model lmul!(1/nobs_used, Σ) lmul!(1/nobs_used, μ) + # at this point μ = μ - μ₀ + # and Σ = Σ + (μ - μ₀)×(μ - μ₀)' - μ₀×μ₀' + mul!(Σ, μ, μ₀', -1, 1) + mul!(Σ, μ₀, μ', -1, 1) mul!(Σ, μ, μ', -1, 1) + μ .+= μ₀ # ridge Σ # while !isposdef(Σ)