From 6da156d6aa98e8976c79398a94d41393ad430e9a Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Tue, 11 Nov 2025 06:49:34 +0200 Subject: [PATCH 1/8] Towards Standard Variables --- src/Common.jl | 2 +- src/Deprecated.jl | 13 +- src/DistributedFactorGraphs.jl | 34 ++-- src/entities/DFGFactor.jl | 2 +- src/entities/DFGVariable.jl | 266 +++++++++----------------- src/services/CompareUtils.jl | 3 - src/services/CustomPrinting.jl | 19 -- src/services/DFGVariable.jl | 328 ++------------------------------- src/services/Serialization.jl | 11 -- test/GraphsDFGSummaryTypes.jl | 6 - test/compareTests.jl | 10 - test/fileDFGTests.jl | 16 -- test/iifCompareTests.jl | 3 +- test/iifInterfaceTests.jl | 6 +- test/interfaceTests.jl | 4 - test/testBlocks.jl | 138 +------------- 16 files changed, 127 insertions(+), 734 deletions(-) diff --git a/src/Common.jl b/src/Common.jl index 8bd1ade9..161c1771 100644 --- a/src/Common.jl +++ b/src/Common.jl @@ -143,7 +143,7 @@ end ## Validation of session, robot, and user labels. ##============================================================================== global _invalidIds = - ["GRAPH", "AGENT", "VARIABLE", "FACTOR", "PPE", "BLOB_ENTRY", "FACTORGRAPH"] + ["GRAPH", "AGENT", "VARIABLE", "FACTOR", "BLOB_ENTRY", "FACTORGRAPH"] const global _validLabelRegex::Regex = r"^[a-zA-Z][-\w\.\@]*$" diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 355d46ae..539d0230 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -52,8 +52,8 @@ function getSolveInProgress( ) # Variable if var isa VariableCompute - if haskey(getSolverDataDict(var), solveKey) - return getSolverDataDict(var)[solveKey].solveInProgress + if haskey(refStates(var), solveKey) + return refStates(var)[solveKey].solveInProgress else return 0 end @@ -148,6 +148,9 @@ function _getDuplicatedEmptyDFG( # DFG.setDescription!(newDfg, "(Copy of) $(DFG.getDescription(dfg))") return newDfg end + +#TODO is Type correct +@deprecate getVariableType(args...) getStateType(args...) ## ================================================================================ ## Deprecated in v0.28 ##================================================================================= @@ -289,7 +292,7 @@ end # Related -# [`listSolveKeys`](@ref), [`getSolverDataDict`](@ref), [`listVariables`](@ref) +# [`listSolveKeys`](@ref), [`refStates`](@ref), [`listVariables`](@ref) # """ function listSolveKeys( variable::VariableCompute, @@ -298,7 +301,7 @@ function listSolveKeys( ) Base.depwarn("listSolveKeys is deprecated, use listStates instead.", :listSolveKeys) # - for ky in keys(getSolverDataDict(variable)) + for ky in keys(refStates(variable)) push!(skeys, ky) end @@ -328,7 +331,7 @@ function listSolveKeys( # skeys = Set{Symbol}() varList = listVariables(dfg, filterVariables; tags = tags, solvable = solvable) - for vs in varList #, ky in keys(getSolverDataDict(getVariable(dfg, vs))) + for vs in varList #, ky in keys(refStates(getVariable(dfg, vs))) listSolveKeys(dfg, vs, filterSolveKeys, skeys) end diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 85cc8886..154b9079 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -379,7 +379,6 @@ const unstable_functions::Vector{Symbol} = [ :getVariableLabelNumber,# TODO somewhat used, do we deprecate? :getfirstBlobentry,# TODO somewhat used, do we deprecate? :getSolveInProgress,#TODO unused, do we deprecate? - :getSolverDataDict,## TODO deprecated or obsolete :hasTagsNeighbors, :isVariable, :isFactor, @@ -433,6 +432,7 @@ const unstable_functions::Vector{Symbol} = [ :setMetadata!, # no set, use add merge :setAgentMetadata!, :setGraphMetadata!, + # :getSolverDataDict,# obsolete #Deprecated in v0.28 :AbstractRelativeMinimize, @@ -469,24 +469,24 @@ const unstable_functions::Vector{Symbol} = [ :PackedFactor, :Factor, :AbstractPointParametricEst, - :MeanMaxPPE, - :getPPEMax, - :getPPEMean, - :getPPESuggested, - :getLastUpdatedTimestamp, - :getPPEDict, - :getVariablePPEDict, - :getVariablePPE, + # :MeanMaxPPE, + # :getPPEMax, + # :getPPEMean, + # :getPPESuggested, + # :getLastUpdatedTimestamp, + # :getPPEDict, + # :getVariablePPEDict, + # :getVariablePPE, :listSolveKeys, :listSupersolves, - :getPPE, - :getPPEs, - :getVariablePPE, - :addPPE!, - :updatePPE!, - :deletePPE!, - :listPPEs, - :mergePPEs!, + # :getPPE, + # :getPPEs, + # :getVariablePPE, + # :addPPE!, + # :updatePPE!, + # :deletePPE!, + # :listPPEs, + # :mergePPEs!, Symbol("@defVariable"), :SmallDataTypes, :NoSolverParams, diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index 998cca0f..702e5a61 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -31,7 +31,7 @@ const FactorCache = AbstractFactorCache # eliminated::Bool = false # TODO should eliminated and potentialused be moved outside of FactorState? potentialused::Bool = false # TODO ^ multihypo::Vector{Float64} = Float64[] # TODO re-evaluate after refactoring w #477 - certainhypo::Vector{Int} = Int[] + certainhypo::Vector{Int} = Int[] #TODO mihgt be dead code? nullhypo::Float64 = 0.0 # solveInProgress::Int = 0 #TODO maybe deprecated or move to operational memory, also why Int? inflation::Float64 = 0.0 diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index 7fb3c80b..d4fa71c6 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -21,10 +21,7 @@ Fields: $(TYPEDFIELDS) """ Base.@kwdef mutable struct State{T <: StateType, P, N} - """ - Globally unique identifier. - """ - id::Union{UUID, Nothing} = nothing # If it's blank it doesn't exist in the DB. + label::Symbol """ Vector of on-manifold points used to represent a ManifoldKernelDensity (or parametric) belief. """ @@ -37,7 +34,6 @@ Base.@kwdef mutable struct State{T <: StateType, P, N} covar::Vector{SMatrix{N, N, Float64}} = SMatrix{getDimension(T), getDimension(T), Float64}[] BayesNetOutVertIDs::Vector{Symbol} = Symbol[] #TODO looks unused? - dimIDs::Vector{Int} = Int[] # TODO Likely deprecate dims::Int = getDimension(T) #TODO should we deprecate in favor of N """ @@ -53,27 +49,23 @@ Base.@kwdef mutable struct State{T <: StateType, P, N} """ Stores the amount information (per measurement dimension) captured in each coordinate dimension. """ - infoPerCoord::Vector{Float64} = zeros(getDimension(T)) + observability::Vector{Float64} = zeros(getDimension(T)) #TODO renamed from infoPerCoord """ Should this variable solveKey be treated as marginalized in inference computations. """ - ismargin::Bool = false + marginalized::Bool = false #TODO renamed from ismargin """ Should this variable solveKey always be kept fluid and not be automatically marginalized. """ dontmargin::Bool = false - # """ - # Convenience flag on whether a solver is currently busy working on this variable solveKey. - # """ - # solveInProgress::Int = 0 """ How many times has a solver updated this variable solveKey estimte. """ - solvedCount::Int = 0 + solves::Int = 0 # TODO renamed from solvedCount """ solveKey identifier associated with this State object. """ - solveKey::Symbol = :default + solveKey::Symbol = :default # TODO replaced by label """ Future proofing field for when more multithreading operations on graph nodes are implemented, these conditions are meant to be used for atomic write transactions to this VND. """ @@ -109,24 +101,23 @@ Fields: $(TYPEDFIELDS) """ Base.@kwdef mutable struct PackedState - id::Union{UUID, Nothing} # If it's blank it doesn't exist in the DB. + label::Symbol vecval::Vector{Float64} dimval::Int vecbw::Vector{Float64} dimbw::Int BayesNetOutVertIDs::Vector{Symbol} # Int - dimIDs::Vector{Int} dims::Int eliminated::Bool # TODO Questionable usage, set but never read? BayesNetVertID::Symbol # Int #TODO deprecate separator::Vector{Symbol} # Int #TODO maybe remove from State and have in variable only. variableType::String initialized::Bool - infoPerCoord::Vector{Float64} - ismargin::Bool + observability::Vector{Float64} # TODO renamed from infoPerCoord + marginalized::Bool # TODO renamed from ismargin dontmargin::Bool - solvedCount::Int - solveKey::Symbol + solves::Int # TODO renamed from solvedCount + solveKey::Symbol #TODO replaced by label covar::Vector{Float64} _version::VersionNumber = _getDFGVersion() end @@ -138,129 +129,21 @@ getLabel(packedstate::PackedState) = packedstate.solveKey # createdTimestamp::DateTime#! # lastUpdatedTimestamp::DateTime#! -##============================================================================== -## PointParametricEst -##============================================================================== - -##------------------------------------------------------------------------------ -## AbstractPointParametricEst interface -##------------------------------------------------------------------------------ - -abstract type AbstractPointParametricEst end - -##------------------------------------------------------------------------------ -## MeanMaxPPE -##------------------------------------------------------------------------------ -""" - $TYPEDEF - -Data container to store Parameteric Point Estimate (PPE) for mean and max. -""" -Base.@kwdef struct MeanMaxPPE <: AbstractPointParametricEst - id::Union{UUID, Nothing} = nothing # If it's blank it doesn't exist in the DB. - solveKey::Symbol - suggested::Vector{Float64} - max::Vector{Float64} - mean::Vector{Float64} - _type::String = "MeanMaxPPE" - _version::VersionNumber = _getDFGVersion() - createdTimestamp::Union{ZonedDateTime, Nothing} = nothing - lastUpdatedTimestamp::Union{ZonedDateTime, Nothing} = nothing -end - -##------------------------------------------------------------------------------ -## Constructors - -function MeanMaxPPE( - solveKey::Symbol, - suggested::Vector{Float64}, - max::Vector{Float64}, - mean::Vector{Float64}, -) - return MeanMaxPPE( - nothing, - solveKey, - suggested, - max, - mean, - "MeanMaxPPE", - _getDFGVersion(), - now(tz"UTC"), - now(tz"UTC"), - ) -end - -## Metadata -""" - $SIGNATURES -Return the fields of MeanMaxPPE that are estimates. -NOTE: This is needed for each AbstractPointParametricEst. -Closest we can get to a decorator pattern. -""" -getEstimateFields(::MeanMaxPPE) = [:suggested, :max, :mean] ##============================================================================== ## DFG Variables ##============================================================================== -""" - $(TYPEDEF) - -The Variable information packed in a way that accomdates multi-lang using json. - -Notes: -- timestamp is a `ZonedDateTime` in UTC. -- nstime can be used as mission time, with the convention that the timestamp millis coincide with the mission start nstime - - e.g. timestamp is `2020-01-01 06:30:01.250 UTC` and first nstime is `250_000_000`. -""" -Base.@kwdef struct VariableDFG <: AbstractGraphVariable - id::Union{UUID, Nothing} = nothing - label::Symbol - tags::Vector{Symbol} = Symbol[] - timestamp::ZonedDateTime = now(tz"UTC") - nstime::String = "0" - ppes::Vector{MeanMaxPPE} = MeanMaxPPE[] - blobEntries::Vector{Blobentry} = Blobentry[] - variableType::String - _version::VersionNumber = _getDFGVersion() - metadata::String = "e30=" - solvable::Int = 1 - solverData::Vector{PackedState} = PackedState[] -end -# maybe add to variable -# createdTimestamp::DateTime -# lastUpdatedTimestamp::DateTime - -#IIF like contruction helper for packed variable -function VariableDFG( - label::Symbol, - variableType::String; - tags::Vector{Symbol} = Symbol[], - timestamp::ZonedDateTime = now(tz"UTC"), - solvable::Int = 1, - nanosecondtime::Int64 = 0, - smalldata::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), - kwargs..., -) - union!(tags, [:VARIABLE]) - - pacvar = VariableDFG(; - label, - variableType, - nstime = string(nanosecondtime), - solvable, - tags, - metadata = base64encode(JSON.json(smalldata)), - timestamp, - kwargs..., - ) - - return pacvar -end - ##------------------------------------------------------------------------------ -## VariableCompute lv2 +## VariableCompute ##------------------------------------------------------------------------------ +# The Variable information packed in a way that accomdates multi-lang using json. + +# Notes: +# - timestamp is a `ZonedDateTime` in UTC. +# - nstime can be used as mission time, with the convention that the timestamp millis coincide with the mission start nstime +# - e.g. timestamp is `2020-01-01 06:30:01.250 UTC` and first nstime is `250_000_000`. + """ $(TYPEDEF) Complete variable structure for a DistributedFactorGraph variable. @@ -269,42 +152,44 @@ Complete variable structure for a DistributedFactorGraph variable. Fields: $(TYPEDFIELDS) """ -Base.@kwdef struct VariableCompute{T <: StateType, P, N} <: AbstractGraphVariable - """The ID for the variable""" - id::Union{UUID, Nothing} = nothing +StructUtils.@kwarg struct VariableDFG{T <: StateType, P, N} <: AbstractGraphVariable + # """The ID for the variable""" + # id::Union{UUID, Nothing} = nothing #NOTE removed in v0.29 """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::ZonedDateTime = now(localzone()) + timestamp::NanoDate = ndnow(UTC) #NOTE changed to NanoDate in v0.29 """Nanoseconds since a user-understood epoch (i.e unix epoch, robot boot time, etc.)""" - nstime::Nanosecond = Nanosecond(0) + steadytime::Union{Nothing, Nanosecond} = nothing #NOTE changed to NanoDate in v0.29 + #nstime::String = "0" #NOTE different uses, as 0-999_999 nanosecond part of timestamp now in timestamp, as steady timestamp now in steadytime """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`getTags`](@ref), [`mergeTags!`](@ref), and [`removeTags!`](@ref)""" tags::Set{Symbol} = Set{Symbol}() - """Dictionary of parametric point estimates keyed by solverDataDict keys - Accessors: [`addPPE!`](@ref), [`updatePPE!`](@ref), and [`deletePPE!`](@ref)""" - ppeDict::Dict{Symbol, AbstractPointParametricEst} = - Dict{Symbol, AbstractPointParametricEst}() - """Dictionary of solver data. May be a subset of all solutions if a solver label was specified in the get call. + # """Dictionary of parametric point estimates keyed by solverDataDict keys + # Accessors: [`addPPE!`](@ref), [`updatePPE!`](@ref), and [`deletePPE!`](@ref)""" + # ppeDict::Dict{Symbol, AbstractPointParametricEst} = + # Dict{Symbol, AbstractPointParametricEst}() #NOTE removed in v0.29 + """Dictionary of state data. May be a subset of all solutions if a solver label was specified in the get call. Accessors: [`addState!`](@ref), [`mergeState!`](@ref), and [`deleteState!`](@ref)""" - solverDataDict::Dict{Symbol, State{T, P, N}} = Dict{Symbol, State{T, P, N}}() + states::OrderedDict{Symbol, State{T, P, N}} = OrderedDict{Symbol, State{T, P, N}}() #NOTE field renamed from solverDataDict in v0.29 """Dictionary of small data associated with this variable. Accessors: [`getBloblet`](@ref), [`setBloblet!`](@ref)""" - smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}() + bloblets::Bloblets = Bloblets() #NOTE changed from smallData in v0.29 """Dictionary of large data associated with this variable. Accessors: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`mergeBlobentry!`](@ref), and [`deleteBlobentry!`](@ref)""" - dataDict::Dict{Symbol, Blobentry} = Dict{Symbol, Blobentry}() + blobentries::Blobentries = Blobentries() #NOTE renamed from dataDict in v0.29 """Solvable flag for the variable. Accessors: [`getSolvable`](@ref), [`setSolvable!`](@ref)""" solvable::Base.RefValue{Int} = Ref(1) + # TODO autotype or version and statetype + _autotype::Nothing = nothing & (name = :type, lower = _ -> TypeMetadata(VariableDFG)) end -refStates(v::VariableCompute) = v.solverDataDict -refMetadata(v::VariableCompute) = v.smallData -refBlobentries(v::VariableCompute) = v.dataDict +refStates(v::VariableCompute) = v.states +const VariableCompute = VariableDFG ##------------------------------------------------------------------------------ ## Constructors @@ -315,10 +200,14 @@ The default VariableCompute constructor. function VariableCompute( label::Symbol, T::Type{<:StateType}; - timestamp::ZonedDateTime = now(localzone()), + timestamp::Union{NanoDate, ZonedDateTime} = ndnow(UTC), solvable::Union{Int, Base.RefValue{Int}} = Ref(1), kwargs..., ) + if timestamp isa ZonedDateTime + # TODO @warn + timestamp = NanoDate(timestamp) + end solvable isa Int && (solvable = Ref(solvable)) N = getDimension(T) @@ -330,14 +219,51 @@ function VariableCompute(label::Symbol, variableType::StateType; kwargs...) return VariableCompute(label, typeof(variableType); kwargs...) end -function VariableCompute(label::Symbol, solverData::State; kwargs...) +function VariableCompute(label::Symbol, state::State; kwargs...) return VariableCompute(; label, - solverDataDict = Dict(:default => solverData), + states = OrderedDict(state.label => state), + kwargs..., + ) +end + +#IIF like contruction helper for VariableDFG +function VariableDFG( + label::Symbol, + stateType::StateType; + tags::Vector{Symbol} = Symbol[], + timestamp::Union{NanoDate, ZonedDateTime} = ndnow(UTC), + solvable::Union{Int, Base.RefValue{Int}} = Ref(1), + nanosecondtime = nothing, + smalldata = nothing, + kwargs..., +) + if timestamp isa ZonedDateTime + # TODO @warn + timestamp = NanoDate(timestamp) + end + if !isnothing(nanosecondtime) + Base.depwarn("nanosecondtime kwarg is deprecated, use steadytime instead", :VariableDFG) + steadytime = Nanosecond(nanosecondtime) + end + if !isnothing(smalldata) + Base.depwarn("smalldata kwarg is deprecated, use bloblets instead", :VariableDFG) + #TODO convert smalldata to bloblets + end + union!(tags, [:VARIABLE]) + + return VariableDFG( + label, + stateType; + steadytime, + solvable, + tags, + timestamp, kwargs..., ) end + # Base.getproperty(x::VariableCompute, f::Symbol) = begin # if f == :solvable # getfield(x, f)[] @@ -367,38 +293,21 @@ Fields: $(TYPEDFIELDS) """ @tags struct VariableSummary <: AbstractGraphVariable - """The ID for the variable""" - id::Union{UUID, Nothing} """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::ZonedDateTime + timestamp::NanoDate """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`getTags`](@ref), [`mergeTags!`](@ref), and [`removeTags!`](@ref)""" tags::Set{Symbol} - """Dictionary of parametric point estimates keyed by solverDataDict keys - Accessors: [`addPPE!`](@ref), [`updatePPE!`](@ref), and [`deletePPE!`](@ref)""" - ppeDict::Dict{Symbol, <:AbstractPointParametricEst} """Symbol for the variableType for the underlying variable. Accessor: [`getVariableType`](@ref)""" variableTypeName::Symbol & (json = (name = "variableType",)) # TODO check from StructTypes.names(::Type{VariableSummary}) = ((:variableTypeName, :variableType),) """Dictionary of large data associated with this variable. Accessors: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`mergeBlobentry!`](@ref), and [`deleteBlobentry!`](@ref)""" - dataDict::Dict{Symbol, Blobentry} -end - -function VariableSummary(id, label, timestamp, tags, ::Nothing, variableTypeName, ::Nothing) - return VariableSummary( - id, - label, - timestamp, - tags, - Dict{Symbol, MeanMaxPPE}(), - variableTypeName, - Dict{Symbol, Blobentry}(), - ) + blobentries::Blobentries end ##------------------------------------------------------------------------------ @@ -414,8 +323,6 @@ Fields: $(TYPEDFIELDS) """ Base.@kwdef struct VariableSkeleton <: AbstractGraphVariable - """The ID for the variable""" - id::Union{UUID, Nothing} = nothing """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol @@ -427,9 +334,8 @@ end function VariableSkeleton( label::Symbol, tags = Set{Symbol}(); - id::Union{UUID, Nothing} = nothing, ) - return VariableSkeleton(id, label, tags) + return VariableSkeleton(label, tags) end ##============================================================================== @@ -438,16 +344,14 @@ end function VariableSummary(v::VariableCompute) return VariableSummary( - v.id, v.label, v.timestamp, copy(v.tags), - deepcopy(v.ppeDict), Symbol(typeof(getVariableType(v))), - v.dataDict, + copy(v.blobentries), ) end function VariableSkeleton(v::AbstractGraphVariable) - return VariableSkeleton(v.id, v.label, copy(v.tags)) + return VariableSkeleton(v.label, copy(v.tags)) end diff --git a/src/services/CompareUtils.jl b/src/services/CompareUtils.jl index 797e9998..b2b1db3d 100644 --- a/src/services/CompareUtils.jl +++ b/src/services/CompareUtils.jl @@ -18,7 +18,6 @@ implement compare if needed. # Generate compares automatically for all in this union const GeneratedCompareUnion = Union{ - MeanMaxPPE, State, PackedState, Blobentry, @@ -200,8 +199,6 @@ function compare(a::State, b::State) a.BayesNetOutVertIDs != b.BayesNetOutVertIDs && @debug("BayesNetOutVertIDs is not equal") === nothing && return false - a.dimIDs != b.dimIDs && @debug("dimIDs is not equal") === nothing && return false - a.dims != b.dims && @debug("dims is not equal") === nothing && return false a.eliminated != b.eliminated && @debug("eliminated is not equal") === nothing && return false diff --git a/src/services/CustomPrinting.jl b/src/services/CustomPrinting.jl index 720c1d53..0c0d520b 100644 --- a/src/services/CustomPrinting.jl +++ b/src/services/CustomPrinting.jl @@ -66,24 +66,6 @@ function printVariable( printstyled(ioc, " VNDs: "; bold = true) println(ioc, solk[smsk], 4 < lsolk ? "..." : "") end - printstyled(ioc, " # PPE solveKeys= ($(length(getPPEDict(vert))))"; bold = true) - println(ioc, "") - if haskey(getPPEDict(vert), :default) - print(ioc, " :default ") - println( - ioc, - "<-- .suggested: ", - round.(getPPE(vert, :default).suggested; digits = 4), - ) - end - maxkeys = 4 - for (key, ppe) in getPPEDict(vert) - key == :default && continue # skip as default is done separately - maxkeys -= 1 - maxkeys == 0 && break - print(ioc, " :$key ") - println(ioc, "<-- .suggested: ", round.(ppe.suggested; digits = 4)) - end println(ioc, " # Blobentries: (", length(listBlobentries(vert)), ")") printstyled(ioc, " VariableType: "; color = :blue, bold = true) println(ioc, vari) @@ -94,7 +76,6 @@ function printVariable( println(ioc, "") :solver in skipfields && push!(skipfields, :solverDataDict) - :ppe in skipfields && push!(skipfields, :ppeDict) t = typeof(vert) fields = setdiff(fieldnames(t), skipfields) diff --git a/src/services/DFGVariable.jl b/src/services/DFGVariable.jl index e8c89b17..304f4116 100644 --- a/src/services/DFGVariable.jl +++ b/src/services/DFGVariable.jl @@ -1,68 +1,24 @@ ##============================================================================== ## Accessors ##============================================================================== -##============================================================================== -## PointParametricEst -##============================================================================== -"$(SIGNATURES)" -getPPEMax(est::AbstractPointParametricEst) = est.max -function getPPEMax(fg::AbstractDFG, varlabel::Symbol, solveKey::Symbol = :default) - return getPPE(fg, varlabel, solveKey) |> getPPEMax -end - -"$(SIGNATURES)" -getPPEMean(est::AbstractPointParametricEst) = est.mean -function getPPEMean(fg::AbstractDFG, varlabel::Symbol, solveKey::Symbol = :default) - return getPPE(fg, varlabel, solveKey) |> getPPEMean -end - -"$(SIGNATURES)" -getPPESuggested(est::AbstractPointParametricEst) = est.suggested -function getPPESuggested(var::VariableCompute, solveKey::Symbol = :default) - return getPPE(var, solveKey) |> getPPESuggested -end -function getPPESuggested(dfg::AbstractDFG, varlabel::Symbol, solveKey::Symbol = :default) - return getPPE(getVariable(dfg, varlabel), solveKey) |> getPPESuggested -end - -"$(SIGNATURES)" -getLastUpdatedTimestamp(est::AbstractPointParametricEst) = est.lastUpdatedTimestamp ##============================================================================== ## Variable Node Data ##============================================================================== -## COMMON -# getSolveInProgress -# isSolveInProgress - ##------------------------------------------------------------------------------ ## variableType ##------------------------------------------------------------------------------ """ $(SIGNATURES) -Variable nodes `variableType` information holding a variety of meta data associated with the type of variable stored in that node of the factor graph. - -Notes -- API Quirk in that this function returns and instance of `::T` not a `::Type{<:StateType}`. - -DevWork -- TODO, see IncrementalInference.jl 1228 - -Related - -getVariableType +Get the type of the variable's state, eg. `Pose2`, `Point3`, etc. as an instance of `StateType`. """ -getVariableType(::VariableCompute{T}) where {T} = T() - -getVariableType(::State{T}) where {T} = T() +getStateType(::VariableCompute{T}) where {T} = T() -# TODO: Confirm that we can switch this out, instead of retrieving the complete variable. -# getVariableType(v::VariableCompute) = getVariableType(getState(v)) +getStateType(::State{T}) where {T} = T() -# Optimized in CGDFG -getVariableType(dfg::AbstractDFG, lbl::Symbol) = getVariableType(getVariable(dfg, lbl)) +getStateType(dfg::AbstractDFG, lbl::Symbol) = getStateType(getVariable(dfg, lbl)) ##------------------------------------------------------------------------------ ## StateType @@ -361,11 +317,11 @@ end ## Variables ##============================================================================== # -# | | label | tags | timestamp | ppe | variableTypeName | solvable | solverData | smallData | dataEntries | -# |---------------------|:-----:|:----:|:---------:|:---:|:----------------:|:--------:|:----------:|:---------:|:-----------:| -# | VariableSkeleton | X | X | | | | | | | | -# | VariableSummary | X | X | X | X | X | | | | X | -# | VariableCompute | X | X | x | X | | X | X | X | X | +# | | label | tags | timestamp | variableTypeName | solvable | solverData | smallData | dataEntries | +# |---------------------|:-----:|:----:|:---------:|:----------------:|:--------:|:----------:|:---------:|:-----------:| +# | VariableSkeleton | X | X | | | | | | | +# | VariableSummary | X | X | X | X | | | | X | +# | VariableCompute | X | X | x | | X | X | X | X | # ##------------------------------------------------------------------------------ @@ -402,74 +358,6 @@ end ## COMMON: -##------------------------------------------------------------------------------ -## ppeDict -##------------------------------------------------------------------------------ - -""" - $SIGNATURES - -Get the PPE dictionary for a variable. Recommended to use CRUD operations instead, [`getPPE`](@ref), [`addPPE!`](@ref), [`updatePPE!`](@ref), [`deletePPE!`](@ref). -""" -getPPEDict(v::AbstractGraphVariable) = v.ppeDict - -#TODO FIXME don't know if this should exist, should rather always update with fg object to simplify inmem vs cloud -""" - $SIGNATURES - -Get the parametric point estimate (PPE) for a variable in the factor graph. - -Notes -- Defaults on keywords `solveKey` and `method` - -Related - -getMeanPPE, getMaxPPE, getKDEMean, getKDEFit, getPPEs, getVariablePPEs -""" -function getPPE(vari::AbstractGraphVariable, solveKey::Symbol = :default) - if haskey(getPPEDict(vari), solveKey) - return getPPEDict(vari)[solveKey] - else - throw(LabelNotFoundError("PPE", solveKey, collect(keys(getPPEDict(vari))))) - end - # return haskey(ppeDict, solveKey) ? ppeDict[solveKey] : nothing -end - -""" - $SIGNATURES - -Get all the parametric point estimate (PPE) for a variable in the factor graph. -""" -function getPPEs end - -# afew more aliases on PPE, brought back from deprecated DF - -""" - $SIGNATURES - -Return full dictionary of PPEs in a variable, recommended to rather use CRUD: [`getPPE`](@ref), -""" -getVariablePPEDict(vari::AbstractGraphVariable) = getPPEDict(vari) - -""" - getVariablePPE(::VariableCompute) - getVariablePPE(::State) - -Get the Parametric Point Estimate of the given variable. -""" -getVariablePPE(args...) = getPPE(args...) - -##------------------------------------------------------------------------------ -## solverDataDict -##------------------------------------------------------------------------------ - -""" - $SIGNATURES - -Get solver data dictionary for a variable. Advised to use graph CRUD operations instead. -""" -getSolverDataDict(v::VariableCompute) = v.solverDataDict - ##------------------------------------------------------------------------------ ## Variable Metadata ##------------------------------------------------------------------------------ @@ -591,14 +479,8 @@ end ##------------------------------------------------------------------------------ function getState(v::VariableCompute, label::Symbol) - !haskey(getSolverDataDict(v), label) && throw(LabelNotFoundError("State", label)) - return getSolverDataDict(v)[label] -end - -function getState(v::VariableDFG, label::Symbol) - stateidx = findfirst(==(label) ∘ getLabel, v.solverData) - isnothing(stateidx) && throw(LabelNotFoundError("State", label)) - return unpackState(v.solverData[stateidx]) + !haskey(refStates(v), label) && throw(LabelNotFoundError("State", label)) + return refStates(v)[label] end """ @@ -789,191 +671,3 @@ function listStates( return labels end -#TODO deprecate PPEs -##============================================================================== -## Point Parametric Estimates -##============================================================================== - -##------------------------------------------------------------------------------ -## CRUD: get, add, update, delete -##------------------------------------------------------------------------------ - -""" - $(SIGNATURES) -Get the parametric point estimate (PPE) for a variable in the factor graph for a given solve key. - -Notes -- Defaults on keywords `solveKey` and `method` - -Related -[`getPPEMean`](@ref), [`getPPEMax`](@ref), [`updatePPE!`](@ref), `mean(BeliefType)` -""" -function getPPE(v::VariableCompute, ppekey::Symbol = :default) - !haskey(v.ppeDict, ppekey) && throw(LabelNotFoundError("PPE", ppekey)) - return v.ppeDict[ppekey] -end -function getPPE(dfg::AbstractDFG, variableLabel::Symbol, ppekey::Symbol = :default) - return getPPE(getVariable(dfg, variableLabel), ppekey) -end -# Not the most efficient call but it at least reuses above (in memory it's probably ok) -function getPPE( - dfg::AbstractDFG, - sourceVariable::AbstractGraphVariable, - ppekey::Symbol = :default, -) - return getPPE(dfg, sourceVariable.label, ppekey) -end - -""" - $(SIGNATURES) -Add variable PPE, errors if it already exists. -""" -function addPPE!( - dfg::AbstractDFG, - variableLabel::Symbol, - ppe::P, -) where {P <: AbstractPointParametricEst} - var = getVariable(dfg, variableLabel) - if haskey(var.ppeDict, ppe.solveKey) - throw(LabelExistsError("PPE", ppe.solveKey)) - end - var.ppeDict[ppe.solveKey] = ppe - return ppe -end - -""" - $(SIGNATURES) -Add a new PPE entry from a deepcopy of the source variable PPE. -NOTE: Copies the PPE. -""" -function addPPE!( - dfg::AbstractDFG, - sourceVariable::VariableCompute, - ppekey::Symbol = :default, -) - return addPPE!(dfg, sourceVariable.label, deepcopy(getPPE(sourceVariable, ppekey))) -end - -function addPPEs!( - dfg::AbstractDFG, - sourceVariables::Vector{VariableCompute}, - ppekey::Symbol = :default, -) - return addPPE!.(dfg, sourceVariables, ppekey) -end - -""" - $(SIGNATURES) -Update PPE data if it exists, otherwise add it -- one call per `key::Symbol=:default`. - -Notes -- uses `ppe.solveKey` as solveKey. -""" -function updatePPE!( - dfg::AbstractDFG, - variableLabel::Symbol, - ppe::AbstractPointParametricEst; - warn_if_absent::Bool = true, -) - var = getVariable(dfg, variableLabel) - if warn_if_absent && !haskey(var.ppeDict, ppe.solveKey) - @warn "PPE '$(ppe.solveKey)' does not exist, adding" - end - #for InMemoryDFGTypes, cloud would update here - var.ppeDict[ppe.solveKey] = ppe - return ppe -end - -""" - $(SIGNATURES) -Update PPE data if it exists, otherwise add it. -NOTE: Copies the PPE data. -""" -function updatePPE!( - dfg::AbstractDFG, - sourceVariable::AbstractGraphVariable, - ppekey::Symbol = :default; - warn_if_absent::Bool = true, -) - return updatePPE!( - dfg, - sourceVariable.label, - deepcopy(getPPE(sourceVariable, ppekey)); - warn_if_absent = warn_if_absent, - ) -end - -""" - $(SIGNATURES) -Update PPE data if it exists, otherwise add it. -""" -function updatePPE!( - dfg::AbstractDFG, - sourceVariables::Vector{<:AbstractGraphVariable}, - ppekey::Symbol = :default; - warn_if_absent::Bool = true, -) - #I think cloud would do this in bulk for speed - for var in sourceVariables - updatePPE!( - dfg, - var.label, - getPPE(dfg, var, ppekey); - warn_if_absent = warn_if_absent, - ) - end -end - -""" - $(SIGNATURES) -Delete PPE data, returns the deleted element. -""" -function deletePPE!(dfg::AbstractDFG, variableLabel::Symbol, ppekey::Symbol = :default) - var = getVariable(dfg, variableLabel) - - if !haskey(var.ppeDict, ppekey) - throw(LabelNotFoundError("PPE", ppekey)) - end - pop!(var.ppeDict, ppekey) - return 1 -end - -""" - $(SIGNATURES) -Delete PPE data, returns the deleted element. -""" -function deletePPE!( - dfg::AbstractDFG, - sourceVariable::VariableCompute, - ppekey::Symbol = :default, -) - return deletePPE!(dfg, sourceVariable.label, ppekey) -end - -##------------------------------------------------------------------------------ -## SET: list, merge -##------------------------------------------------------------------------------ - -""" - $(SIGNATURES) -List all the PPE data keys in the variable. -""" -function listPPEs(dfg::AbstractDFG, variableLabel::Symbol) - v = getVariable(dfg, variableLabel) - return collect(keys(v.ppeDict))::Vector{Symbol} -end - -#TODO API and only correct level -""" - $(SIGNATURES) -Merges and updates solver and estimate data for a variable (variable can be from another graph). -Note: Makes a copy of the estimates and solver data so that there is no coupling between graphs. -""" -function mergePPEs!( - destVariable::AbstractGraphVariable, - sourceVariable::AbstractGraphVariable, -) - # We don't know which graph this came from, must be copied! - merge!(destVariable.ppeDict, deepcopy(sourceVariable.ppeDict)) - return destVariable -end diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index 640bc302..bbc9ebd7 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -81,8 +81,6 @@ function packState(d::State{T}) where {T <: StateType} d.bw[:], size(d.bw, 1), d.BayesNetOutVertIDs, - d.dimIDs, - d.dims, d.eliminated, d.BayesNetVertID, d.separator, @@ -132,8 +130,6 @@ function unpackState(d::PackedState) #TODO only one covar is currently supported in packed VND covar = isempty(d.covar) ? SMatrix{N, N, Float64}[] : [d.covar], BayesNetOutVertIDs = Symbol.(d.BayesNetOutVertIDs), - dimIDs = d.dimIDs, - dims = d.dims, eliminated = d.eliminated, BayesNetVertID = Symbol(d.BayesNetVertID), separator = Symbol.(d.separator), @@ -154,7 +150,6 @@ end function packVariable( v::VariableCompute; - includePPEs::Bool = true, includeSolveData::Bool = true, includeDataEntries::Bool = true, ) @@ -164,7 +159,6 @@ function packVariable( timestamp = v.timestamp, nstime = string(v.nstime.value), tags = collect(v.tags), # Symbol.() - ppes = collect(values(v.ppeDict)), solverData = packState.(collect(values(v.solverDataDict))), metadata = base64encode(JSON.json(v.smallData)), solvable = getSolvable(v), @@ -176,7 +170,6 @@ end function packVariable( v::VariableDFG; - includePPEs::Bool = true, includeSolveData::Bool = true, includeDataEntries::Bool = true, ) @@ -193,9 +186,6 @@ function unpackVariable(variable::VariableDFG; skipVersionCheck::Bool = false) ) pointType = DFG.getPointType(variableType) - ppeDict = - Dict{Symbol, MeanMaxPPE}(map(p -> p.solveKey, variable.ppes) .=> variable.ppes) - N = getDimension(variableType) solverDict = Dict{Symbol, State{variableType, pointType, N}}( map(sd -> sd.solveKey, variable.solverData) .=> @@ -213,7 +203,6 @@ function unpackVariable(variable::VariableDFG; skipVersionCheck::Bool = false) timestamp = variable.timestamp, nstime = Nanosecond(variable.nstime), tags = Set(variable.tags), - ppeDict = ppeDict, solverDataDict = solverDict, smallData = metadata, dataDict = dataDict, diff --git a/test/GraphsDFGSummaryTypes.jl b/test/GraphsDFGSummaryTypes.jl index b13a458a..1b8e3c9b 100644 --- a/test/GraphsDFGSummaryTypes.jl +++ b/test/GraphsDFGSummaryTypes.jl @@ -9,7 +9,6 @@ function DistributedFactorGraphs.VariableSummary(label::Symbol) label, DistributedFactorGraphs.now(localzone()), Set{Symbol}(), - Dict{Symbol, MeanMaxPPE}(), :Pose2, Dict{Symbol, Blobentry}(), ) @@ -21,7 +20,6 @@ function DistributedFactorGraphs.VariableSummary(label::Symbol, ::State{T}) wher label, DistributedFactorGraphs.now(localzone()), Set{Symbol}(), - Dict{Symbol, MeanMaxPPE}(), Symbol(T), Dict{Symbol, Blobentry}(), ) @@ -111,10 +109,6 @@ end end end -@testset "Updating Nodes" begin - VARTYPE == VariableSummary && PPETestBlock!(dfg, v1) -end - @testset "Adjacency Matrices" begin fg = GraphsDFG{NoSolverParams, VARTYPE, FACTYPE}() addVariable!(fg, VARTYPE(:a)) diff --git a/test/compareTests.jl b/test/compareTests.jl index 4d254305..3af8c3ea 100644 --- a/test/compareTests.jl +++ b/test/compareTests.jl @@ -20,16 +20,6 @@ vnd2.val[1] = [0.1;] @test !(vnd1 == vnd2) @test !(vnd1 == vnd3) -# MeanMaxPPE -ppe1 = MeanMaxPPE(:default, [1.0], [2.0], [3.0]) -ppe2 = deepcopy(ppe1) -ppe3 = MeanMaxPPE(:default, [2.0], [3.0], [4.0]) - -@test ppe1 == ppe2 -ppe2.max[1] = 0.1 -@test !(ppe1 == ppe2) -@test !(ppe1 == ppe3) - # VariableCompute v1 = VariableCompute(:x1, TestVariableType1()) v2 = deepcopy(v1) diff --git a/test/fileDFGTests.jl b/test/fileDFGTests.jl index 3c181d2d..96fad868 100644 --- a/test/fileDFGTests.jl +++ b/test/fileDFGTests.jl @@ -49,18 +49,6 @@ using UUIDs verts, ) - # Add some PPEs - ppe1 = MeanMaxPPE(:default, [1.0], [0.2], [3.456]) - ppe2 = MeanMaxPPE(; - solveKey = :other, - suggested = [1.0], - mean = [0.2], - max = [3.456], - createdTimestamp = ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"), - ) - map(v -> addPPE!(dfg, getLabel(v), deepcopy(ppe1)), verts) - map(v -> addPPE!(dfg, getLabel(v), deepcopy(ppe2)), verts) - #call update to set it on cloud mergeVariable!.(dfg, verts) @@ -136,10 +124,6 @@ using UUIDs for v in ls(dfg) @test getBlobentries(getVariable(dfg, v)) == getBlobentries(getVariable(retDFG, v)) - @test issetequal(listPPEs(dfg, v), listPPEs(retDFG, v)) - for ppe in listPPEs(dfg, v) - @test getPPE(dfg, v, ppe) == getPPE(retDFG, v, ppe) - end end # test the duplicate order #581 diff --git a/test/iifCompareTests.jl b/test/iifCompareTests.jl index 9daf5e1c..0cdb5cff 100644 --- a/test/iifCompareTests.jl +++ b/test/iifCompareTests.jl @@ -55,7 +55,7 @@ using Test fg, fg2, skipsamples = true, - skip = Symbol[:infoPerCoord; :initialized; :inferdim; :ppeDict; :solvedCount], + skip = Symbol[:infoPerCoord; :initialized; :inferdim; :solvedCount], ) # fg2 has been solved, so it should fail on the estimate dictionary @test !compareSimilarVariables( @@ -87,7 +87,6 @@ using Test :infoPerCoord :initialized :inferdim - :ppeDict :solvedCount ], ) diff --git a/test/iifInterfaceTests.jl b/test/iifInterfaceTests.jl index 79a2baec..a5dd1bc6 100644 --- a/test/iifInterfaceTests.jl +++ b/test/iifInterfaceTests.jl @@ -177,12 +177,8 @@ end @test getLabel(v1) == v1.label @test getTags(v1) == v1.tags @test getTimestamp(v1) == v1.timestamp - @test getVariablePPEDict(v1) == v1.ppeDict - @test_throws LabelNotFoundError DistributedFactorGraphs.getVariablePPE(v1, :notfound) @test getState(v1, :default) === v1.solverDataDict[:default] - @test getSolverDataDict(v1) == v1.solverDataDict - # legacy compat test - @test getVariablePPEDict(v1) == v1.ppeDict # changed to .ppeDict -- delete by DFG v0.7 + @test refStates(v1) == v1.solverDataDict @test typeof(getVariableType(v1)) == Position{1} @test typeof(getVariableType(v2)) == Position{1} diff --git a/test/interfaceTests.jl b/test/interfaceTests.jl index d41fafec..46adc894 100644 --- a/test/interfaceTests.jl +++ b/test/interfaceTests.jl @@ -111,10 +111,6 @@ end tagsTestBlock!(fg1, var1, v1_tags) end -@testset "Parametric Point Estimates" begin - PPETestBlock!(fg1, var1) -end - @testset "Variable Solver Data" begin VSDTestBlock!(fg1, var1) end diff --git a/test/testBlocks.jl b/test/testBlocks.jl index 7c7aad0f..bf402448 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -88,7 +88,6 @@ function DFGStructureAndAccessors( :AGENT, :VARIABLE, :FACTOR, - :PPE, :BLOB_ENTRY, :FACTORGRAPH, ] @@ -291,9 +290,7 @@ function DFGVariableSCA() @test getSolvable(v2) == 1 # TODO direct use is not recommended, use accessors, maybe not export or deprecate - @test getSolverDataDict(v1) == v1.solverDataDict - - @test getPPEDict(v1) == v1.ppeDict + @test refStates(v1) == v1.solverDataDict # @test getMetadata(v1) == Dict{Symbol, MetadataTypes}() @@ -320,10 +317,7 @@ function DFGVariableSCA() @test getManifold(testvar) == TranslationGroup(1) # #TODO sort out - # getPPEs # getState - # getVariablePPEs - # getVariablePPE # getSolvedCount # isSolved # setSolvedCount @@ -565,132 +559,6 @@ function tagsTestBlock!(fg, v1, v1_tags) @test !hasTagsNeighbors(fg, :abf1, [:LANDMARK, :TAG]) end -function PPETestBlock!(fg, v1) - # "Parametric Point Estimates" - - # - `getPPEs` - # **Set** - # > - `emptyPPE!` - # > - `mergePPE!` - - # Add a new PPE of type MeanMaxPPE to :x0 - ppe = MeanMaxPPE(:default, [0.0], [0.0], [0.0]) - - @test getPPEMax(ppe) === ppe.max - @test getPPEMean(ppe) === ppe.mean - @test getPPESuggested(ppe) === ppe.suggested - @test getLastUpdatedTimestamp(ppe) === ppe.lastUpdatedTimestamp - - @test addPPE!(fg, :a, ppe) == ppe - @test_throws LabelExistsError addPPE!(fg, :a, ppe) - - @test listPPEs(fg, :a) == [:default] - - # Get the data back - note that this is a reference to above. - @test getPPE(getVariable(fg, :a), :default) == ppe - @test getPPE(fg, :a, :default) == ppe - @test getPPEMean(fg, :a, :default) == ppe.mean - @test getPPEMax(fg, :a, :default) == ppe.max - @test getPPESuggested(fg, :a, :default) == ppe.suggested - - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - - @test_throws LabelNotFoundError getPPE(fg, :a, :default) - # Update add it - @test @test_logs (:warn, Regex("'$(ppe.solveKey)' does not exist")) match_mode = :any updatePPE!( - fg, - :a, - ppe, - ) == ppe - # Update update it - @test updatePPE!(fg, :a, ppe) == ppe - @test deletePPE!(fg, :a, :default) == 1 - - # manually add ppe to v1 for tests - v1.ppeDict[:default] = deepcopy(ppe) - # Bulk copy PPE's for :x1 - @test updatePPE!(fg, [v1], :default) == nothing - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - - # New interface - @test addPPE!(fg, :a, ppe) == ppe - # Update update it - @test updatePPE!(fg, :a, ppe) == ppe - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - - #FIXME copied from lower - # @test @test_deprecated getVariablePPEs(v1) == v1.ppeDict - @test_throws LabelNotFoundError getPPE(v1, :notfound) - #TODO - # @test_deprecated getVariablePPE(v1) - - # Add a new PPE of type MeanMaxPPE to :x0 - ppe = MeanMaxPPE(:default, [0.0], [0.0], [0.0]) - addPPE!(fg, :a, ppe) - @test listPPEs(fg, :a) == [:default] - # Get the data back - note that this is a reference to above. - @test getPPE(fg, :a, :default) == ppe - - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - # Update add it - updatePPE!(fg, :a, ppe) #, :default) - # Update update it - updatePPE!(fg, :a, ppe) #, :default) - - v1.ppeDict[:default] = deepcopy(ppe) - # Bulk copy PPE's for x0 and x1 - updatePPE!(fg, [v1], :default) - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - - #TODO DEPRECATE - # getEstimates - # estimates - # getVariablePPEs - # getVariablePPE - - # newvar = deepcopy(v1) - # getPPEDict(newvar)[:default] = MeanMaxPPE(:default, [150.0], [100.0], [50.0]) - # @test !(getPPEDict(newvar) == getPPEDict(v1)) - # delete!(getVariablePPEs(newvar), :default) - # getVariablePPEs(newvar)[:second] = MeanMaxPPE(:second, [15.0], [10.0], [5.0]) - # @test symdiff(collect(keys(getVariablePPEs(v1))), [:default, :second]) == Symbol[] - # @test symdiff(collect(keys(getVariablePPEs(newvar))), [:second]) == Symbol[] - # # Get the source too. - # @test symdiff(collect(keys(getVariablePPEs(getVariable(dfg, :a)))), [:default, :second]) == Symbol[] - #update - - ## TODO make sure these are covered - # global dfg - # #get the variable - # var1 = getVariable(dfg, :a) - # #make a copy and simulate external changes - # newvar = deepcopy(var1) - # getVariablePPEs(newvar)[:default] = MeanMaxPPE(:default, [150.0], [100.0], [50.0]) - # #update - # mergeUpdateVariableSolverData!(dfg, newvar) - # #For now spot check - # # @test solverDataDict(newvar) == solverDataDict(var1) - # @test getVariablePPEs(newvar) == getVariablePPEs(var1) - # - # # Delete :default and replace to see if new ones can be added - # delete!(getVariablePPEs(newvar), :default) - # getVariablePPEs(newvar)[:second] = MeanMaxPPE(:second, [15.0], [10.0], [5.0]) - # - # # Persist to the original variable. - # mergeUpdateVariableSolverData!(dfg, newvar) - # # At this point newvar will have only :second, and var1 should have both (it is the reference) - # @test symdiff(collect(keys(getVariablePPEs(var1))), [:default, :second]) == Symbol[] - # @test symdiff(collect(keys(getVariablePPEs(newvar))), [:second]) == Symbol[] - # # Get the source too. - # @test symdiff(collect(keys(getVariablePPEs(getVariable(dfg, :a)))), [:default, :second]) == Symbol[] - ## -end - function VSDTestBlock!(fg, v1) # "Variable Solver Data" # #### Variable Solver Data @@ -769,7 +637,7 @@ function VSDTestBlock!(fg, v1) return nothing #TODO solverDataDict() not deprecated - # @test getSolverDataDict(newvar) == getSolverDataDict(v1) + # @test refStates(newvar) == refStates(v1) # @test @test_deprecated mergeUpdateVariableSolverData!(fg, newvar) @@ -1682,8 +1550,6 @@ function FileDFGTestBlock(testDFGAPI; kwargs...) vnd.BayesNetVertID = :outid push!(vnd.BayesNetOutVertIDs, :id) # vnd.bw[1] = [1.0;] - push!(vnd.dimIDs, 1) - vnd.dims = 1 vnd.dontmargin = true vnd.eliminated = true vnd.infoPerCoord .= Float64[1.5;] From bbde17c04e0fefe1317d6c075f680c45cc60928d Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Tue, 11 Nov 2025 07:59:55 +0200 Subject: [PATCH 2/8] Unused State eliminated, BayesNetOutVertIDs,BayesNetVertID --- src/entities/DFGVariable.jl | 24 ++++++++++-------------- src/services/CompareUtils.jl | 18 +++++++++--------- src/services/Serialization.jl | 12 ++++++------ test/testBlocks.jl | 6 +++--- 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index d4fa71c6..2df3b186 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -33,14 +33,14 @@ Base.@kwdef mutable struct State{T <: StateType, P, N} "Parametric (Gaussian) covariance." covar::Vector{SMatrix{N, N, Float64}} = SMatrix{getDimension(T), getDimension(T), Float64}[] - BayesNetOutVertIDs::Vector{Symbol} = Symbol[] #TODO looks unused? + # BayesNetOutVertIDs::Vector{Symbol} = Symbol[] #TODO looks unused? dims::Int = getDimension(T) #TODO should we deprecate in favor of N - """ - Flag used by junction (Bayes) tree construction algorithm to know whether this variable has yet been included in the tree construction. - """ - eliminated::Bool = false - BayesNetVertID::Symbol = :NOTHING # Union{Nothing, } #TODO deprecate + # """ + # Flag used by junction (Bayes) tree construction algorithm to know whether this variable has yet been included in the tree construction. + # """ + # eliminated::Bool = false + # BayesNetVertID::Symbol = :NOTHING # Union{Nothing, } #TODO deprecate separator::Vector{Symbol} = Symbol[] """ False if initial numerical values are not yet available or stored values are not ready for further processing yet. @@ -106,10 +106,10 @@ Base.@kwdef mutable struct PackedState dimval::Int vecbw::Vector{Float64} dimbw::Int - BayesNetOutVertIDs::Vector{Symbol} # Int + # BayesNetOutVertIDs::Vector{Symbol} # Int dims::Int - eliminated::Bool # TODO Questionable usage, set but never read? - BayesNetVertID::Symbol # Int #TODO deprecate + # eliminated::Bool # TODO Questionable usage, set but never read? + # BayesNetVertID::Symbol # Int #TODO deprecate separator::Vector{Symbol} # Int #TODO maybe remove from State and have in variable only. variableType::String initialized::Bool @@ -187,7 +187,7 @@ StructUtils.@kwarg struct VariableDFG{T <: StateType, P, N} <: AbstractGraphVari _autotype::Nothing = nothing & (name = :type, lower = _ -> TypeMetadata(VariableDFG)) end -refStates(v::VariableCompute) = v.states +refStates(v::VariableDFG) = v.states const VariableCompute = VariableDFG ##------------------------------------------------------------------------------ @@ -215,10 +215,6 @@ function VariableCompute( return VariableCompute{T, P, N}(; label, timestamp, solvable, kwargs...) end -function VariableCompute(label::Symbol, variableType::StateType; kwargs...) - return VariableCompute(label, typeof(variableType); kwargs...) -end - function VariableCompute(label::Symbol, state::State; kwargs...) return VariableCompute(; label, diff --git a/src/services/CompareUtils.jl b/src/services/CompareUtils.jl index b2b1db3d..ad09b675 100644 --- a/src/services/CompareUtils.jl +++ b/src/services/CompareUtils.jl @@ -196,15 +196,15 @@ end function compare(a::State, b::State) a.val != b.val && @debug("val is not equal") === nothing && return false a.bw != b.bw && @debug("bw is not equal") === nothing && return false - a.BayesNetOutVertIDs != b.BayesNetOutVertIDs && - @debug("BayesNetOutVertIDs is not equal") === nothing && - return false - a.eliminated != b.eliminated && - @debug("eliminated is not equal") === nothing && - return false - a.BayesNetVertID != b.BayesNetVertID && - @debug("BayesNetVertID is not equal") === nothing && - return false + # a.BayesNetOutVertIDs != b.BayesNetOutVertIDs && + # @debug("BayesNetOutVertIDs is not equal") === nothing && + # return false + # a.eliminated != b.eliminated && + # @debug("eliminated is not equal") === nothing && + # return false + # a.BayesNetVertID != b.BayesNetVertID && + # @debug("BayesNetVertID is not equal") === nothing && + # return false a.separator != b.separator && @debug("separator is not equal") === nothing && return false diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index bbc9ebd7..3c0ef55c 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -80,9 +80,9 @@ function packState(d::State{T}) where {T <: StateType} size(castval, 1), d.bw[:], size(d.bw, 1), - d.BayesNetOutVertIDs, - d.eliminated, - d.BayesNetVertID, + # d.BayesNetOutVertIDs, + # d.eliminated, + # d.BayesNetVertID, d.separator, stringVariableType(getVariableType(d)), d.initialized, @@ -129,9 +129,9 @@ function unpackState(d::PackedState) bw = BW, #TODO only one covar is currently supported in packed VND covar = isempty(d.covar) ? SMatrix{N, N, Float64}[] : [d.covar], - BayesNetOutVertIDs = Symbol.(d.BayesNetOutVertIDs), - eliminated = d.eliminated, - BayesNetVertID = Symbol(d.BayesNetVertID), + # BayesNetOutVertIDs = Symbol.(d.BayesNetOutVertIDs), + # eliminated = d.eliminated, + # BayesNetVertID = Symbol(d.BayesNetVertID), separator = Symbol.(d.separator), initialized = d.initialized, infoPerCoord = d.infoPerCoord, diff --git a/test/testBlocks.jl b/test/testBlocks.jl index bf402448..d1c10001 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -1547,11 +1547,11 @@ function FileDFGTestBlock(testDFGAPI; kwargs...) v4 = getVariable(dfg, :x4) vnd = getState(v4, :default) # set everything - vnd.BayesNetVertID = :outid - push!(vnd.BayesNetOutVertIDs, :id) + # vnd.BayesNetVertID = :outid + # push!(vnd.BayesNetOutVertIDs, :id) # vnd.bw[1] = [1.0;] vnd.dontmargin = true - vnd.eliminated = true + # vnd.eliminated = true vnd.infoPerCoord .= Float64[1.5;] vnd.initialized = true vnd.ismargin = true From 940898ab3bbbf1fcc9bd67ad9dce69b905d15110 Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Wed, 12 Nov 2025 16:40:14 +0200 Subject: [PATCH 3/8] use TimesDates and Variable serialization WIP --- Project.toml | 4 +- src/DataBlobs/entities/BlobEntry.jl | 35 ++++- src/DataBlobs/services/BlobEntry.jl | 72 +++++----- src/Deprecated.jl | 2 +- src/DistributedFactorGraphs.jl | 5 +- src/FileDFG/services/FileDFG.jl | 18 +-- src/entities/Bloblet.jl | 43 ++++++ src/entities/DFGFactor.jl | 23 ++-- src/entities/DFGVariable.jl | 196 ++++++++++++++-------------- src/services/AbstractDFG.jl | 2 +- src/services/CompareUtils.jl | 13 +- src/services/CustomPrinting.jl | 13 +- src/services/DFGVariable.jl | 33 ++--- src/services/Serialization.jl | 142 +++++++++++--------- test/fileDFGTests.jl | 8 +- test/iifInterfaceTests.jl | 4 +- test/interfaceTests.jl | 2 +- test/testBlocks.jl | 28 ++-- 18 files changed, 362 insertions(+), 281 deletions(-) diff --git a/Project.toml b/Project.toml index f5e849ab..82902fd9 100644 --- a/Project.toml +++ b/Project.toml @@ -16,7 +16,6 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb" -NanoDates = "46f1a544-deae-4307-8689-c12aa3c955c6" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" @@ -29,6 +28,7 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b" TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" +TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [weakdeps] @@ -57,7 +57,6 @@ JSON = "1.0.0" LieGroups = "0.1" LinearAlgebra = "1.10" ManifoldsBase = "1, 2" -NanoDates = "1.0.3" OrderedCollections = "1.4" Pkg = "1.4, 1.5" ProgressMeter = "1" @@ -72,6 +71,7 @@ Tar = "1.9" TensorCast = "0.3.3, 0.4" Test = "1.10" TimeZones = "1.3.1" +TimesDates = "0.3.3" UUIDs = "1.10" julia = "1.10" diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index 20df8da4..cbb849c0 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -40,18 +40,21 @@ StructUtils.@kwarg struct Blobentry """ Additional information that can help a different user of the Blob. """ description::String = "" """ MIME description describing the format of binary data in the `Blob`, e.g. 'image/png' or 'application/json'. """ - mimetype::String = "application/octet-stream" #FIXME ::MIME = MIME("application/octet-stream") + mimetype::MIME = MIME("application/octet-stream") """ Storage for a couple of bytes directly in the graph. Use with caution and keep it small and simple.""" metadata::JSONText = JSONText("{}") """ When the Blob itself was first created. Serialized as an ISO 8601 string.""" - timestamp::NanoDate = ndnow(UTC) & (json = (lower = timestamp,),) + timestamp::TimeDateZone = TimeDateZone(now(localzone())) """ Type version of this Blobentry.""" - version::VersionNumber = version(Blobentry) + version::VersionNumber = DFG.version(Blobentry) end version(::Type{Blobentry}) = v"0.1.0" -function Blobentry(label::Symbol, blobstore = :default; kwargs...) - return Blobentry(; label, blobstore, kwargs...) +function Blobentry(label::Symbol, blobstore = :default; metadata, kwargs...) + if !(metadata isa JSONText) + metadata = JSONText(JSON.json(metadata)) + end + return Blobentry(; label, blobstore, metadata, kwargs...) end # construction helper from existing Blobentry for user overriding via kwargs function Blobentry( @@ -123,3 +126,25 @@ function Base.setproperty!(x::Blobentry, f::Symbol, val) end const Blobentries = OrderedDict{Symbol, Blobentry} + +#TODO write Blobentries as array +# StructUtils.dictlike(::Type{Blobentries}) = false +# StructUtils.structlike(::Type{Blobentries}) = false +# StructUtils.arraylike(::Type{Blobentries}) = false + +# function StructUtils.lower(entries::Blobentries) +# return map(collect(values(entries))) do (entry) +# return StructUtils.lower(entry) +# end +# end + +# function StructUtils.lift(s::StructUtils.StructStyle, T::Type{Blobentries}, json_vector) +# entries = T() +# @warn "lifting" s T json_vector +# foreach(json_vector) do obj +# global gobj = obj +# return push!(entries, Symbol(obj.label) => StructUtils.make(s, Blobentry, obj)) +# end +# @warn "lifted Blobentries" entries +# return entries, nothing +# end diff --git a/src/DataBlobs/services/BlobEntry.jl b/src/DataBlobs/services/BlobEntry.jl index b97b003f..9a1a1926 100644 --- a/src/DataBlobs/services/BlobEntry.jl +++ b/src/DataBlobs/services/BlobEntry.jl @@ -67,17 +67,17 @@ Also see: [`addBlobentry!`](@ref), [`getBlob`](@ref), [`listBlobentries`](@ref) """ function getBlobentry(var::AbstractGraphVariable, key::Symbol) if !hasBlobentry(var, key) - throw(LabelNotFoundError("Blobentry", key, collect(keys(var.dataDict)))) + throw(LabelNotFoundError("Blobentry", key, collect(keys(var.blobentries)))) end - return var.dataDict[key] + return var.blobentries[key] end -function getBlobentry(var::VariableDFG, key::Symbol) - if !hasBlobentry(var, key) - throw(LabelNotFoundError("Blobentry", key)) - end - return var.blobEntries[findfirst(x -> x.label == key, var.blobEntries)] -end +# function getBlobentry(var::VariableDFG, key::Symbol) +# if !hasBlobentry(var, key) +# throw(LabelNotFoundError("Blobentry", key)) +# end +# return var.blobEntries[findfirst(x -> x.label == key, var.blobEntries)] +# end """ $(SIGNATURES) @@ -131,17 +131,17 @@ Should be extended if DFG variable is not returned by reference. Also see: [`getBlobentry`](@ref), [`addBlob!`](@ref), [`mergeBlobentry!`](@ref) """ function addBlobentry!(var::VariableCompute, entry::Blobentry) - haskey(var.dataDict, entry.label) && throw(LabelExistsError("Blobentry", entry.label)) - var.dataDict[entry.label] = entry + haskey(var.blobentries, entry.label) && throw(LabelExistsError("Blobentry", entry.label)) + var.blobentries[entry.label] = entry return entry end -function addBlobentry!(var::VariableDFG, entry::Blobentry) - entry.label in getproperty.(var.blobEntries, :label) && - throw(LabelExistsError("Blobentry", entry.label)) - push!(var.blobEntries, entry) - return entry -end +# function addBlobentry!(var::VariableDFG, entry::Blobentry) +# entry.label in getproperty.(var.blobEntries, :label) && +# throw(LabelExistsError("Blobentry", entry.label)) +# push!(var.blobEntries, entry) +# return entry +# end function addBlobentry!(dfg::AbstractDFG, vLbl::Symbol, entry::Blobentry) return addBlobentry!(getVariable(dfg, vLbl), entry) @@ -158,10 +158,10 @@ If the Blobentry does not exist, it will be added. Notes: """ function mergeBlobentry!(var::AbstractGraphVariable, bde::Blobentry) - if !haskey(var.dataDict, bde.label) + if !haskey(var.blobentries, bde.label) addBlobentry!(var, bde) else - var.dataDict[bde.label] = bde + var.blobentries[bde.label] = bde end return 1 end @@ -179,15 +179,15 @@ Notes: """ function deleteBlobentry!(var::VariableCompute, key::Symbol) !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key)) - delete!(var.dataDict, key) + delete!(var.blobentries, key) return 1 end -function deleteBlobentry!(var::VariableDFG, key::Symbol) - !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key)) - deleteat!(var.blobEntries, findfirst(x -> x.label == key, var.blobEntries)) - return 1 -end +# function deleteBlobentry!(var::VariableDFG, key::Symbol) +# !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key)) +# deleteat!(var.blobEntries, findfirst(x -> x.label == key, var.blobEntries)) +# return 1 +# end function deleteBlobentry!(dfg::AbstractDFG, label::Symbol, key::Symbol) return deleteBlobentry!(getVariable(dfg, label), key) @@ -223,11 +223,11 @@ end Does a blob entry exist with `blobLabel`. """ -hasBlobentry(v::VariableCompute, blobLabel::Symbol) = haskey(v.dataDict, blobLabel) +hasBlobentry(v::VariableCompute, blobLabel::Symbol) = haskey(v.blobentries, blobLabel) -function hasBlobentry(v::VariableDFG, label::Symbol) - return label in getproperty.(v.blobEntries, :label) -end +# function hasBlobentry(v::VariableDFG, label::Symbol) +# return label in getproperty.(v.blobEntries, :label) +# end """ $(SIGNATURES) @@ -235,12 +235,12 @@ end Get blob entries, returns a `Vector{Blobentry}`. """ function getBlobentries(v::VariableCompute) - return collect(values(v.dataDict)) + return collect(values(v.blobentries)) end -function getBlobentries(v::VariableDFG) - return copy(v.blobEntries) -end +# function getBlobentries(v::VariableDFG) +# return copy(v.blobEntries) +# end function getBlobentries( v::AbstractGraphVariable; @@ -289,12 +289,12 @@ const collectBlobentries = gatherBlobentries List the blob entries associated with a particular variable. """ function listBlobentries(var::AbstractGraphVariable) - return collect(keys(var.dataDict)) + return collect(keys(var.blobentries)) end -function listBlobentries(var::VariableDFG) - return getproperty.(var.blobEntries, :label) -end +# function listBlobentries(var::VariableDFG) +# return getproperty.(var.blobEntries, :label) +# end function listBlobentries(dfg::AbstractDFG, label::Symbol) return listBlobentries(getVariable(dfg, label)) diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 539d0230..783a9a5c 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -193,7 +193,7 @@ function setSolverData!(v::VariableCompute, data::State, key::Symbol = :default) :setSolverData!, ) @assert key == data.solveKey "State.solveKey=:$(data.solveKey) does not match requested :$(key)" - return v.solverDataDict[key] = data + return v.states[key] = data end @deprecate mergeVariableSolverData!(args...; kwargs...) mergeState!(args...; kwargs...) diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 154b9079..9910c3e5 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -20,7 +20,7 @@ using DocStringExtensions using Dates using Random using TimeZones -using NanoDates +using TimesDates using JSON export StructUtils # export for use in macros using LinearAlgebra @@ -83,7 +83,7 @@ export FactorDFG, FactorSummary, FactorSkeleton export Blobentry -export State, PackedState +export State ##------------------------------------------------------------------------------ ## Functions @@ -531,6 +531,7 @@ include("entities/Bloblet.jl") include("DataBlobs/entities/BlobEntry.jl") include("DataBlobs/entities/BlobStores.jl") +include("serialization/DFGStructStyles.jl") include("serialization/PackedSerialization.jl") include("serialization/DistributionSerialization.jl") diff --git a/src/FileDFG/services/FileDFG.jl b/src/FileDFG/services/FileDFG.jl index f66c215b..7cc9bfe4 100644 --- a/src/FileDFG/services/FileDFG.jl +++ b/src/FileDFG/services/FileDFG.jl @@ -40,19 +40,19 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG; saveMetadata::Bool = map(f -> rm("$factorFolder/$f"), readdir(factorFolder)) # Variables @showprogress "saving variables" for v in variables - vPacked = packVariable(v) - JSON.json("$varFolder/$(v.label).json", vPacked) + # vPacked = packVariable(v) + JSON.json("$varFolder/$(v.label).json", v; style = DFGJSONStyle()) end # Factors @showprogress "saving factors" for f in factors - JSON.json("$factorFolder/$(f.label).json", f) + JSON.json("$factorFolder/$(f.label).json", f; style = DFGJSONStyle()) end #GraphsDFG metadata if saveMetadata @assert isa(dfg, GraphsDFG) "only metadata for GraphsDFG are supported" @info "saving dfg metadata" fgPacked = GraphsDFGs.packDFGMetadata(dfg) - JSON.json("$savepath/dfg.json", fgPacked) + JSON.json("$savepath/dfg.json", fgPacked; style = DFGJSONStyle()) end savedir = dirname(savepath) # is this a path of just local name? #344 -- workaround with unique names @@ -138,7 +138,7 @@ function loadDFG!( @assert isa(dfgLoadInto, GraphsDFG) "Only GraphsDFG metadata are supported" @info "loading dfg metadata" jstr = read("$folder/dfg.json", String) - fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG) + fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG; style = DFGJSONStyle()) GraphsDFGs.unpackDFGMetadata!(dfgLoadInto, fgPacked) end @@ -164,7 +164,7 @@ function loadDFG!( # type instability on `variables` as either `::Vector{Variable}` or `::Vector{VariableCompute{<:}}` (vector of abstract) variables = @showprogress 1 "loading variables" asyncmap(varFiles) do varFile jstr = read("$varFolder/$varFile", String) - packedvar = JSON.parse(jstr, VariableDFG) + packedvar = JSON.parse(jstr, VariableDFG; style = DFGJSONStyle()) v = usePackedVariable ? packedvar : unpackVariable(packedvar) return addVariable!(dfgLoadInto, v) end @@ -176,7 +176,7 @@ function loadDFG!( # `factors` is not type stable `::Vector{Factor}` or `::Vector{FactorCompute{<:}}` (vector of abstract) factors = @showprogress 1 "loading factors" asyncmap(factorFiles) do factorFile - f = JSON.parsefile("$factorFolder/$factorFile", FactorDFG) + f = JSON.parsefile("$factorFolder/$factorFile", FactorDFG; style = DFGJSONStyle()) return addFactor!(dfgLoadInto, f) end @@ -228,12 +228,12 @@ function loadDFG(file::AbstractString) #TODO deprecate old format, v0.28 local fgPacked try - fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG) + fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG; style = DFGJSONStyle()) catch e if e isa MethodError @warn "Deprecated serialization: Failed to read DFG metadata. Attempting to load using the old format. Error:" e fgPacked = - GraphsDFGs.PackedGraphsDFG(JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG)) + GraphsDFGs.PackedGraphsDFG(JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG; style = DFGJSONStyle())) else rethrow(e) end diff --git a/src/entities/Bloblet.jl b/src/entities/Bloblet.jl index ff724f17..2eec982c 100644 --- a/src/entities/Bloblet.jl +++ b/src/entities/Bloblet.jl @@ -21,3 +21,46 @@ function StructUtils.lift(::Type{Bloblets}, json_vector::Vector) Symbol(x["label"]) => Bloblet(Symbol(x["label"]), x["val"]) for x in json_vector ) end + +""" + $(SIGNATURES) +""" +function getBloblet(node, label::Symbol) + !haskey(refBloblets(node), label) && throw(LabelNotFoundError("Bloblet", label)) + return refBloblets(node)[label] +end + +""" + $(SIGNATURES) +""" +function addBloblet!(node, bloblet::Bloblet) + label = getLabel(bloblet) + haskey(refBloblets(node), label) && throw(LabelExistsError("Bloblet", label)) + refBloblets(node)[label] = bloblet + return bloblet +end + +""" + $(SIGNATURES) +""" +function mergeBloblet!(node, bloblet::Bloblet) + refBloblets(node)[getLabel(bloblet)] = bloblet + return 1 +end + +""" + $(SIGNATURES) +""" +function deleteBloblet!(node, label::Symbol) + !haskey(refBloblets(node), label) && throw(LabelNotFoundError("Bloblet", label)) + pop!(refBloblets(node), label) + return 1 +end + +""" + $(SIGNATURES) +List all Bloblet keys for a variable `label` in `dfg` +""" +function listBloblets(node) + return collect(keys(refBloblets(node))) +end diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index 702e5a61..3ec25715 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -74,13 +74,13 @@ StructUtils.@kwarg struct FactorDFG{T <: AbstractObservation, N} <: AbstractGrap variableorder::NTuple{N, Symbol} & (choosetype = x->NTuple{length(x), Symbol},) # NOTE v0.29 renamed from _variableOrderSymbols """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::NanoDate = ndnow(UTC) & (lower = timestamp,) # NOTE v0.29 changed from ZonedDateTime + timestamp::TimeDateZone = TimeDateZone(now(localzone())) # NOTE v0.29 changed from ZonedDateTime # TODO # """(Optional) Steady (monotonic) time in nanoseconds `Nanosecond` (`Int64``)""" # nstime::Nanosecond #NOTE v0.29 REMOVED as not used, add when needed, or now as steadytime. """Solvable flag for the factor. Accessors: [`getSolvable`](@ref), [`setSolvable!`](@ref)""" - solvable::Base.RefValue{Int} = Ref(1) & (lower = getindex, lift = Ref) + solvable::Base.RefValue{Int} = Ref(1) #& (lower = getindex, lift = Ref) """Dictionary of small data associated with this variable. Accessors: [`getBloblet`](@ref), [`addBloblet!`](@ref)""" bloblets::Bloblets = Bloblets() #NOTE v0.29 changed from smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}() @@ -102,16 +102,13 @@ end version(::Type{<:FactorDFG}) = v"0.29.0" -#FIXME use style to avoid type piracy -StructUtils.structlike(::JSON.JSONStyle, ::Type{Base.RefValue{Int64}}) = false - ##------------------------------------------------------------------------------ ## Constructors function FactorDFG( variableorder::Union{<:Tuple, Vector{Symbol}}, observation::AbstractObservation; label::Symbol = assembleFactorName(variableorder), - timestamp::Union{NanoDate, ZonedDateTime} = ndnow(UTC), + timestamp::Union{TimeDateZone, ZonedDateTime} = TimeDateZone(now(localzone())), tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}([:FACTOR]), bloblets::Bloblets = Bloblets(), multihypo::Vector{Float64} = Float64[], @@ -132,10 +129,10 @@ function FactorDFG( if timestamp isa ZonedDateTime Base.depwarn( - "`FactorDFG` timestamp as `ZonedDateTime` is deprecated, use `NanoDate` instead", + "`FactorDFG` timestamp as `ZonedDateTime` is deprecated, use `TimeDateZone` instead", :FactorDFG, ) - timestamp = NanoDate(timestamp.utc_datetime) + timestamp = TimeDateZone(timestamp.utc_datetime) end # create factor data @@ -165,7 +162,7 @@ function FactorDFG( state::FactorState = FactorState(), cache = nothing; tags::Set{Symbol} = Set{Symbol}([:FACTOR]), - timestamp::Union{DateTime, ZonedDateTime, NanoDate} = ndnow(UTC), + timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = TimeDateZone(now(localzone())), solvable::Int = 1, bloblets::Bloblets = Bloblets(), blobentries::Blobentries = Blobentries(), @@ -192,10 +189,10 @@ function FactorDFG( # deprecated in v0.29 if timestamp isa ZonedDateTime Base.depwarn( - "`FactorDFG` timestamp as `ZonedDateTime` is deprecated, use `NanoDate(timestamp.utc_datetime)` instead", + "`FactorDFG` timestamp as `ZonedDateTime` is deprecated, use `TimeDateZone(timestamp.utc_datetime)` instead", :FactorDFG, ) - nd_timestamp = NanoDate(timestamp.utc_datetime) + nd_timestamp = TimeDateZone(timestamp.utc_datetime) else nd_timestamp = timestamp end @@ -239,13 +236,13 @@ Base.@kwdef struct FactorSummary <: AbstractGraphFactor variableorder::Tuple{Vararg{Symbol}} #TODO changed to NTuple """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::NanoDate + timestamp::TimeDateZone end function FactorSummary( label::Symbol, variableorder::Union{Vector{Symbol}, Tuple}; - timestamp::NanoDate = ndnow(UTC), + timestamp::TimeDateZone = TimeDateZone(now(localzone())), tags::Set{Symbol} = Set{Symbol}(), ) return FactorSummary(label, tags, Tuple(variableorder), timestamp) diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index 2df3b186..d1e41188 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -20,8 +20,11 @@ N: Manifold dimension. Fields: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct State{T <: StateType, P, N} - label::Symbol +@kwdef mutable struct State{T <: StateType, P, N} + """ + Identifier associated with this State object. + """ + label::Symbol # TODO renamed from solveKey """ Vector of on-manifold points used to represent a ManifoldKernelDensity (or parametric) belief. """ @@ -35,7 +38,7 @@ Base.@kwdef mutable struct State{T <: StateType, P, N} SMatrix{getDimension(T), getDimension(T), Float64}[] # BayesNetOutVertIDs::Vector{Symbol} = Symbol[] #TODO looks unused? - dims::Int = getDimension(T) #TODO should we deprecate in favor of N + # dims::Int = getDimension(T) #TODO should we deprecate in favor of N # """ # Flag used by junction (Bayes) tree construction algorithm to know whether this variable has yet been included in the tree construction. # """ @@ -49,28 +52,25 @@ Base.@kwdef mutable struct State{T <: StateType, P, N} """ Stores the amount information (per measurement dimension) captured in each coordinate dimension. """ - observability::Vector{Float64} = zeros(getDimension(T)) #TODO renamed from infoPerCoord + observability::Vector{Float64} = Float64[]#zeros(getDimension(T)) #TODO renamed from infoPerCoord """ Should this variable solveKey be treated as marginalized in inference computations. """ marginalized::Bool = false #TODO renamed from ismargin - """ - Should this variable solveKey always be kept fluid and not be automatically marginalized. - """ - dontmargin::Bool = false + # """ + # Should this variable solveKey always be kept fluid and not be automatically marginalized. + # """ + # dontmargin::Bool = false """ How many times has a solver updated this variable solveKey estimte. """ solves::Int = 0 # TODO renamed from solvedCount - """ - solveKey identifier associated with this State object. - """ - solveKey::Symbol = :default # TODO replaced by label - """ - Future proofing field for when more multithreading operations on graph nodes are implemented, these conditions are meant to be used for atomic write transactions to this VND. - """ - events::Dict{Symbol, Threads.Condition} = Dict{Symbol, Threads.Condition}() + # """ + # Future proofing field for when more multithreading operations on graph nodes are implemented, these conditions are meant to be used for atomic write transactions to this VND. + # """ + # events::Dict{Symbol, Threads.Condition} = Dict{Symbol, Threads.Condition}() # + statetype::Symbol = Symbol(stringVariableType(T())) end ##------------------------------------------------------------------------------ @@ -78,8 +78,8 @@ end function State{T}(; kwargs...) where {T <: StateType} return State{T, getPointType(T), getDimension(T)}(; kwargs...) end -function State(variableType::StateType; kwargs...) - return State{typeof(variableType)}(; kwargs...) +function State(label, variableType::StateType; kwargs...) + return State{typeof(variableType)}(; label, kwargs...) end function State(state::State; kwargs...) @@ -88,47 +88,32 @@ function State(state::State; kwargs...) kwargs..., ) end -##============================================================================== -## PackedState.jl -##============================================================================== -""" -$(TYPEDEF) -Packed State structure for serializing DFGVariables. +StructUtils.structlike(::Type{<:State}) = false +StructUtils.lower(state::State) = DFG.packState(state) +StructUtils.lift(::Type{<:State}, obj) = DFG.unpackState(obj) - --- -Fields: -$(TYPEDFIELDS) -""" -Base.@kwdef mutable struct PackedState - label::Symbol - vecval::Vector{Float64} - dimval::Int - vecbw::Vector{Float64} - dimbw::Int - # BayesNetOutVertIDs::Vector{Symbol} # Int - dims::Int - # eliminated::Bool # TODO Questionable usage, set but never read? - # BayesNetVertID::Symbol # Int #TODO deprecate - separator::Vector{Symbol} # Int #TODO maybe remove from State and have in variable only. - variableType::String - initialized::Bool - observability::Vector{Float64} # TODO renamed from infoPerCoord - marginalized::Bool # TODO renamed from ismargin - dontmargin::Bool - solves::Int # TODO renamed from solvedCount - solveKey::Symbol #TODO replaced by label - covar::Vector{Float64} - _version::VersionNumber = _getDFGVersion() -end +##------------------------------------------------------------------------------ +## States - OrderedDict{Symbol, State} +const States = OrderedDict{Symbol, State{T, P, N}} where {T<:AbstractStateType,P,N} -#FIXME remove once solveKey field is renamed to `label` -getLabel(packedstate::PackedState) = packedstate.solveKey +StructUtils.dictlike(::Type{<:States}) = false +StructUtils.structlike(::Type{<:States}) = false +StructUtils.arraylike(::Type{<:States}) = false -# maybe add -# createdTimestamp::DateTime#! -# lastUpdatedTimestamp::DateTime#! +function StructUtils.lower(states::States) + return map(collect(values(states))) do (state) + return StructUtils.lower(state) + end +end +function StructUtils.lift(::StructUtils.StructStyle, S::Type{<:States{T}}, json_vector) where T + states = S() + foreach(json_vector) do obj + return push!(states, Symbol(obj.label) => StructUtils.make(State{T}, obj)) + end + return states, nothing +end ##============================================================================== ## DFG Variables @@ -152,25 +137,19 @@ Complete variable structure for a DistributedFactorGraph variable. Fields: $(TYPEDFIELDS) """ -StructUtils.@kwarg struct VariableDFG{T <: StateType, P, N} <: AbstractGraphVariable - # """The ID for the variable""" - # id::Union{UUID, Nothing} = nothing #NOTE removed in v0.29 +@kwdef struct VariableDFG{T <: StateType, P, N} <: AbstractGraphVariable """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::NanoDate = ndnow(UTC) #NOTE changed to NanoDate in v0.29 + timestamp::TimeDateZone = TimeDateZone(now(localzone())) #NOTE changed to TimeDateZone in v0.29 """Nanoseconds since a user-understood epoch (i.e unix epoch, robot boot time, etc.)""" - steadytime::Union{Nothing, Nanosecond} = nothing #NOTE changed to NanoDate in v0.29 + steadytime::Union{Nothing, Nanosecond} = nothing #NOTE changed to TimeDateZone in v0.29 #nstime::String = "0" #NOTE different uses, as 0-999_999 nanosecond part of timestamp now in timestamp, as steady timestamp now in steadytime """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`getTags`](@ref), [`mergeTags!`](@ref), and [`removeTags!`](@ref)""" tags::Set{Symbol} = Set{Symbol}() - # """Dictionary of parametric point estimates keyed by solverDataDict keys - # Accessors: [`addPPE!`](@ref), [`updatePPE!`](@ref), and [`deletePPE!`](@ref)""" - # ppeDict::Dict{Symbol, AbstractPointParametricEst} = - # Dict{Symbol, AbstractPointParametricEst}() #NOTE removed in v0.29 """Dictionary of state data. May be a subset of all solutions if a solver label was specified in the get call. Accessors: [`addState!`](@ref), [`mergeState!`](@ref), and [`deleteState!`](@ref)""" states::OrderedDict{Symbol, State{T, P, N}} = OrderedDict{Symbol, State{T, P, N}}() #NOTE field renamed from solverDataDict in v0.29 @@ -182,63 +161,69 @@ StructUtils.@kwarg struct VariableDFG{T <: StateType, P, N} <: AbstractGraphVari blobentries::Blobentries = Blobentries() #NOTE renamed from dataDict in v0.29 """Solvable flag for the variable. Accessors: [`getSolvable`](@ref), [`setSolvable!`](@ref)""" - solvable::Base.RefValue{Int} = Ref(1) + solvable::Base.RefValue{Int} = Ref(1) #& (lower = getindex,) + statetype::Symbol = Symbol(stringVariableType(T())) # TODO autotype or version and statetype - _autotype::Nothing = nothing & (name = :type, lower = _ -> TypeMetadata(VariableDFG)) + _autotype::Nothing = nothing #& (name = :type, lower = _ -> TypeMetadata(VariableDFG)) end - +version(::Type{<:VariableDFG}) = v"0.29" refStates(v::VariableDFG) = v.states +#NOTE fielddefaults and fieldtags not through @kwarg macro due to error with State{T, P, N} +function StructUtils.fielddefaults(::StructUtils.StructStyle, ::Type{VariableDFG{T,P,N}}) where {T,P,N} + return ( + timestamp = TimeDateZone(now(localzone())), + tags = Set{Symbol}(), + # states = OrderedDict{Symbol, State{T, P, N}}(), + bloblets = Bloblets(), + blobentries = Blobentries(), + solvable = Ref(1), + _autotype = nothing, + ) +end + +function StructUtils.fieldtags(::StructUtils.StructStyle, ::Type{<:VariableDFG}) + return ( + _autotype = (name = :type, lower = _ -> TypeMetadata(VariableDFG)), + # solvable = (lower = getindex,), + ) +end + +function resolveVariableType(lazyobj) + T = parseVariableType(lazyobj.statetype[]) + return VariableDFG{T, getPointType(T), getDimension(T)} +end + +@choosetype VariableDFG resolveVariableType + +# JSON.omit_empty(::DistributedFactorGraphs.DFGJSONStyle, ::Type{<:VariableDFG}) = true + const VariableCompute = VariableDFG ##------------------------------------------------------------------------------ ## Constructors """ $SIGNATURES -The default VariableCompute constructor. +The default VariableDFG constructor. """ -function VariableCompute( - label::Symbol, - T::Type{<:StateType}; - timestamp::Union{NanoDate, ZonedDateTime} = ndnow(UTC), - solvable::Union{Int, Base.RefValue{Int}} = Ref(1), - kwargs..., -) - if timestamp isa ZonedDateTime - # TODO @warn - timestamp = NanoDate(timestamp) - end - solvable isa Int && (solvable = Ref(solvable)) - - N = getDimension(T) - P = getPointType(T) - return VariableCompute{T, P, N}(; label, timestamp, solvable, kwargs...) -end - -function VariableCompute(label::Symbol, state::State; kwargs...) - return VariableCompute(; - label, - states = OrderedDict(state.label => state), - kwargs..., - ) -end - #IIF like contruction helper for VariableDFG function VariableDFG( label::Symbol, - stateType::StateType; - tags::Vector{Symbol} = Symbol[], - timestamp::Union{NanoDate, ZonedDateTime} = ndnow(UTC), + stateType::Union{T, Type{T}}; + tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}(), + timestamp::Union{TimeDateZone, ZonedDateTime} = TimeDateZone(now(localzone())), solvable::Union{Int, Base.RefValue{Int}} = Ref(1), + steadytime::Union{Nothing, Nanosecond} = nothing, nanosecondtime = nothing, smalldata = nothing, kwargs..., -) +) where {T <: StateType} if timestamp isa ZonedDateTime # TODO @warn - timestamp = NanoDate(timestamp) + timestamp = TimeDateZone(timestamp) end if !isnothing(nanosecondtime) + @assert isnothing(steadytime), "nanosecondtime is replaced by steadytime. Cannot specify both steadytime and nanosecondtime" Base.depwarn("nanosecondtime kwarg is deprecated, use steadytime instead", :VariableDFG) steadytime = Nanosecond(nanosecondtime) end @@ -248,9 +233,10 @@ function VariableDFG( end union!(tags, [:VARIABLE]) - return VariableDFG( + N = getDimension(T) + P = getPointType(T) + return VariableDFG{T, P, N}(; label, - stateType; steadytime, solvable, tags, @@ -259,6 +245,14 @@ function VariableDFG( ) end +function VariableDFG(label::Symbol, state::State; kwargs...) + return VariableCompute(; + label, + states = OrderedDict(state.label => state), + kwargs..., + ) +end + # Base.getproperty(x::VariableCompute, f::Symbol) = begin # if f == :solvable @@ -294,7 +288,7 @@ $(TYPEDFIELDS) label::Symbol """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::NanoDate + timestamp::TimeDateZone """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`getTags`](@ref), [`mergeTags!`](@ref), and [`removeTags!`](@ref)""" tags::Set{Symbol} diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 982f285c..7c8e96ee 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -441,7 +441,7 @@ function getVariable(dfg::AbstractDFG, label::Symbol, solveKey::Symbol) # function getVariable(dfg::AbstractDFG, label::Symbol; stateLabelFilter::Union{Nothing, ...} = nothing) var = getVariable(dfg, label) - if isa(var, VariableCompute) && !haskey(var.solverDataDict, solveKey) + if isa(var, VariableCompute) && !haskey(var.states, solveKey) throw(LabelNotFoundError("VariableNode", solveKey)) elseif !isa(var, VariableCompute) @warn "getVariable(dfg, label, solveKey) only supported for type VariableCompute." diff --git a/src/services/CompareUtils.jl b/src/services/CompareUtils.jl index ad09b675..f82675b9 100644 --- a/src/services/CompareUtils.jl +++ b/src/services/CompareUtils.jl @@ -19,7 +19,6 @@ implement compare if needed. # Generate compares automatically for all in this union const GeneratedCompareUnion = Union{ State, - PackedState, Blobentry, VariableCompute, VariableDFG, @@ -211,13 +210,13 @@ function compare(a::State, b::State) a.initialized != b.initialized && @debug("initialized is not equal") === nothing && return false - !isapprox(a.infoPerCoord, b.infoPerCoord; atol = 1e-13) && + !isapprox(a.observability, b.observability; atol = 1e-13) && @debug("infoPerCoord is not equal") === nothing && return false - a.ismargin != b.ismargin && @debug("ismargin is not equal") === nothing && return false - a.dontmargin != b.dontmargin && - @debug("dontmargin is not equal") === nothing && - return false + a.marginalized != b.marginalized && @debug("ismargin is not equal") === nothing && return false + # a.dontmargin != b.dontmargin && + # @debug("dontmargin is not equal") === nothing && + # return false getVariableType(a) != getVariableType(b) && @debug("variableType is not equal") === nothing && return false @@ -252,7 +251,7 @@ function compareVariable( varskiplist = skipsamples ? [:val; :bw] : Symbol[] skiplist = union([:variableType;], varskiplist) union!(skiplist, skip) - TP = TP && compareAll(A.solverDataDict, B.solverDataDict; skip = skiplist, show = show) + TP = TP && compareAll(A.states, B.states; skip = skiplist, show = show) Ad = getState(A, :default) #FIXME why onlly comparing default? Bd = getState(B, :default) diff --git a/src/services/CustomPrinting.jl b/src/services/CustomPrinting.jl index 0c0d520b..08a346b6 100644 --- a/src/services/CustomPrinting.jl +++ b/src/services/CustomPrinting.jl @@ -17,9 +17,9 @@ function printVariable( if short # opmemt = (getVariableType(vert) |> typeof ).name - vari = getVariableType(vert) |> typeof - printstyled(ioc, typeof(vert).name.name, "{"; bold = true) - printstyled(ioc, vari.name.name; bold = true, color = :blue) + vari = getStateType(vert) |> typeof + printstyled(ioc, nameof((typeof(vert))), "{"; bold = true) + printstyled(ioc, vari; bold = true, color = :blue) printstyled(ioc, "...}"; bold = true) println(ioc, "") # printstyled(ioc, summary(vert),"\n", bold=true) @@ -30,14 +30,13 @@ function printVariable( println(ioc, "") catch e end - vnd = if haskey(vert.solverDataDict, :default) + vnd = if hasState(vert, :default) getState(vert, :default) else nothing end - println(ioc, " ID: ", vert.id) println(ioc, " timestamp: ", vert.timestamp) - println(ioc, " nstime: ", vert.nstime) + isnothing(vert.steadytime) || println(ioc, " steadytime: ", vert.steadytime) print(ioc, " label: ") printstyled(ioc, vert.label; bold = true) println(ioc) @@ -49,7 +48,7 @@ function printVariable( # list the marginalization status ismarg = solk .|> x -> isMarginalized(vert, x) isinit = solk .|> x -> isInitialized(vert, x) - printstyled(ioc, " # VND solveKeys= ($(lsolk))"; bold = true) + printstyled(ioc, " # states: ($(lsolk))"; bold = true) println(ioc, "") printstyled(ioc, " # initialized: "; bold = true) println(ioc, "(true=", sum(isinit), ",false=", length(isinit) - sum(isinit), ")") diff --git a/src/services/DFGVariable.jl b/src/services/DFGVariable.jl index 304f4116..8ac47546 100644 --- a/src/services/DFGVariable.jl +++ b/src/services/DFGVariable.jl @@ -116,8 +116,8 @@ end Interface function to return the `<:ManifoldsBase.AbstractManifold` object of `variableType<:StateType`. """ getManifold(::T) where {T <: StateType} = getManifold(T) -getManifold(vari::VariableCompute) = getVariableType(vari) |> getManifold -getManifold(state::State) = getVariableType(state) |> getManifold +getManifold(vari::VariableCompute) = getStateType(vari) |> getManifold +getManifold(state::State) = getStateType(state) |> getManifold # covers both <:StateType and <:AbstractObservation getManifold(dfg::AbstractDFG, lbl::Symbol) = getManifold(dfg[lbl]) @@ -212,7 +212,7 @@ Related isSolved, setSolvedCount! """ -getSolvedCount(v::State) = v.solvedCount +getSolvedCount(v::State) = v.solves function getSolvedCount(v::VariableCompute, solveKey::Symbol = :default) return getState(v, solveKey) |> getSolvedCount end @@ -229,7 +229,7 @@ Related getSolved, isSolved """ -setSolvedCount!(v::State, val::Int) = v.solvedCount = val +setSolvedCount!(v::State, val::Int) = v.solves = val function setSolvedCount!(v::VariableCompute, val::Int, solveKey::Symbol = :default) return setSolvedCount!(getState(v, solveKey), val) end @@ -251,7 +251,7 @@ Related getSolved, setSolved! """ -isSolved(v::State) = 0 < v.solvedCount +isSolved(v::State) = 0 < v.solves function isSolved(v::VariableCompute, solveKey::Symbol = :default) return getState(v, solveKey) |> isSolved end @@ -287,7 +287,7 @@ Notes: - State default `solveKey=:default` """ function isMarginalized(vert::VariableCompute, solveKey::Symbol = :default) - return getState(vert, solveKey).ismargin + return getState(vert, solveKey).marginalized end function isMarginalized(dfg::AbstractDFG, sym::Symbol, solveKey::Symbol = :default) return isMarginalized(DFG.getVariable(dfg, sym), solveKey) @@ -299,7 +299,7 @@ end Mark a variable as marginalized `true` or `false`. """ function setMarginalized!(vnd::State, val::Bool) - return vnd.ismargin = val + return vnd.marginalized = val end function setMarginalized!(vari::VariableCompute, val::Bool, solveKey::Symbol = :default) return setMarginalized!(getState(vari, solveKey), val) @@ -477,6 +477,7 @@ end ##------------------------------------------------------------------------------ ## CRUD: get, add, update, delete ##------------------------------------------------------------------------------ +hasState(v::VariableCompute, label::Symbol) = haskey(v.states, label) function getState(v::VariableCompute, label::Symbol) !haskey(refStates(v), label) && throw(LabelNotFoundError("State", label)) @@ -494,7 +495,7 @@ end function getStates(dfg::AbstractDFG, variableLabel::Symbol) v = getVariable(dfg, variableLabel) - return collect(values(v.solverDataDict)) + return collect(values(v.states)) end """ @@ -507,10 +508,10 @@ function addState!(dfg::GraphsDFG, variableLabel::Symbol, state::State) end function addState!(v::VariableCompute, state::State) - if haskey(v.solverDataDict, state.solveKey) - throw(LabelExistsError("State", state.solveKey)) + if haskey(v.states, state.label) + throw(LabelExistsError("State", state.label)) end - v.solverDataDict[state.solveKey] = state + v.states[state.label] = state return state end @@ -546,10 +547,10 @@ function mergeState!(dfg::GraphsDFG, variableLabel::Symbol, vnd::State) end function mergeState!(v::VariableCompute, vnd::State) - if !haskey(v.solverDataDict, vnd.solveKey) + if !haskey(v.states, vnd.label) addState!(v, vnd) else - v.solverDataDict[vnd.solveKey] = vnd + v.states[vnd.label] = vnd end return 1 end @@ -596,10 +597,10 @@ function deleteState!(dfg::GraphsDFG, variableLabel::Symbol, label::Symbol) end function deleteState!(v::VariableCompute, label::Symbol) - if !haskey(v.solverDataDict, label) + if !haskey(v.states, label) throw(LabelNotFoundError("State", label)) end - delete!(v.solverDataDict, label) + delete!(v.states, label) return 1 end @@ -637,7 +638,7 @@ end List all the variable state labels. """ function listStates(v::VariableCompute; labelFilter::Union{Nothing, Function} = nothing) - labels = collect(keys(v.solverDataDict)) + labels = collect(keys(v.states)) return filterDFG!(labels, labelFilter) end diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index 3c0ef55c..f562173b 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -42,7 +42,6 @@ function parseVariableType(_typeString::AbstractString) if isnothing(subtype) throw(SerializationError("Unable to deserialize type $(_typeString), not found")) - return nothing end if isnothing(param) @@ -57,10 +56,31 @@ end ##============================================================================== ## State Packing and unpacking ##============================================================================== - -# returns a PackedState +# Old PackedState struct fields +# id::Union{UUID, Nothing} +# vecval::Vector{Float64} +# dimval::Int +# vecbw::Vector{Float64} +# dimbw::Int +# BayesNetOutVertIDs::Vector{Symbol} +# dimIDs::Vector{Int} +# dims::Int +# eliminated::Bool +# BayesNetVertID::Symbol +# separator::Vector{Symbol} +# variableType::String +# initialized::Bool +# infoPerCoord::Vector{Float64} +# ismargin::Bool +# dontmargin::Bool +# solvedCount::Int +# solveKey::Symbol +# covar::Vector{Float64} +# _version::VersionNumber = _getDFGVersion() + +# returns a named tuple until State serialization is fully consolidated function packState(d::State{T}) where {T <: StateType} - @debug "Dispatching conversion variable -> packed variable for type $(string(getVariableType(d)))" + @debug "Dispatching conversion variable -> packed variable for type $(string(getStateType(d)))" castval = if 0 < length(d.val) precast = getCoordinates.(T, d.val) @cast castval[i, j] := precast[j][i] @@ -73,40 +93,62 @@ function packState(d::State{T}) where {T <: StateType} length(d.covar) > 1 && @warn( "Packing of more than one parametric covariance is NOT supported yet, only packing first." ) - - return PackedState( - d.id, - _val, - size(castval, 1), - d.bw[:], - size(d.bw, 1), - # d.BayesNetOutVertIDs, - # d.eliminated, - # d.BayesNetVertID, - d.separator, - stringVariableType(getVariableType(d)), - d.initialized, - d.infoPerCoord, - d.ismargin, - d.dontmargin, - # d.solveInProgress, - d.solvedCount, - d.solveKey, - isempty(d.covar) ? Float64[] : vec(d.covar[1]), - _getDFGVersion(), + + return ( + label = d.label, + vecval =_val, + dimval = size(castval, 1), + vecbw = d.bw[:], + dimbw = size(d.bw, 1), + separator = d.separator, + statetype = stringVariableType(getStateType(d)), + initialized = d.initialized, + observability = d.observability, + marginalized = d.marginalized, + solves = d.solves, + covar = isempty(d.covar) ? Float64[] : vec(d.covar[1]), + version = version(State), ) end -function unpackState(d::PackedState) +function unpackOldState(d) @debug "Dispatching conversion packed variable -> variable for type $(string(d.variableType))" # Figuring out the variableType - # TODO deprecated remove in v0.11 - for backward compatibility for saved variableTypes. - ststring = string(split(d.variableType, "(")[1]) - T = parseVariableType(ststring) - isnothing(T) && error( - "The variable doesn't seem to have a variableType. It needs to set up with an StateType from IIF. This will happen if you use DFG to add serialized variables directly and try use them. Please use IncrementalInference.addVariable().", + T = parseVariableType(d.variableType) + + r3 = d.dimval + c3 = r3 > 0 ? floor(Int, length(d.vecval) / r3) : 0 + M3 = reshape(d.vecval, r3, c3) + @cast val_[j][i] := M3[i, j] + vals = Vector{getPointType(T)}(undef, length(val_)) + # vals = getPoint.(T, val_) + for (i, v) in enumerate(val_) + vals[i] = getPoint(T, v) + end + + r4 = d.dimbw + c4 = r4 > 0 ? floor(Int, length(d.vecbw) / r4) : 0 + BW = reshape(d.vecbw, r4, c4) + + # + N = getDimension(T) + return State{T, getPointType(T), N}(; + label = Symbol(d.label), + val = vals, + bw = BW, + #TODO only one covar is currently supported in packed VND + covar = isempty(d.covar) ? SMatrix{N, N, Float64}[] : [d.covar], + separator = Symbol.(d.separator), + initialized = d.initialized, + observability = d.infoPerCoord, + marginalized = d.ismargin, + solves = d.solvedCount, ) +end +function unpackState(d) + @debug "Dispatching conversion packed variable -> variable for type $(string(d.statetype))" + T = parseVariableType(d.statetype) r3 = d.dimval c3 = r3 > 0 ? floor(Int, length(d.vecval) / r3) : 0 M3 = reshape(d.vecval, r3, c3) @@ -124,23 +166,16 @@ function unpackState(d::PackedState) # N = getDimension(T) return State{T, getPointType(T), N}(; - id = d.id, + label = Symbol(d.label), val = vals, bw = BW, #TODO only one covar is currently supported in packed VND covar = isempty(d.covar) ? SMatrix{N, N, Float64}[] : [d.covar], - # BayesNetOutVertIDs = Symbol.(d.BayesNetOutVertIDs), - # eliminated = d.eliminated, - # BayesNetVertID = Symbol(d.BayesNetVertID), separator = Symbol.(d.separator), initialized = d.initialized, - infoPerCoord = d.infoPerCoord, - ismargin = d.ismargin, - dontmargin = d.dontmargin, - # solveInProgress = d.solveInProgress, - solvedCount = d.solvedCount, - solveKey = Symbol(d.solveKey), - events = Dict{Symbol, Threads.Condition}(), + observability = d.observability, + marginalized = d.marginalized, + solves = d.solves, ) end @@ -153,30 +188,22 @@ function packVariable( includeSolveData::Bool = true, includeDataEntries::Bool = true, ) - return VariableDFG(; + return ( id = v.id, label = v.label, timestamp = v.timestamp, nstime = string(v.nstime.value), tags = collect(v.tags), # Symbol.() - solverData = packState.(collect(values(v.solverDataDict))), + solverData = packState.(collect(values(v.states))), metadata = base64encode(JSON.json(v.smallData)), solvable = getSolvable(v), variableType = stringVariableType(DFG.getVariableType(v)), - blobEntries = collect(values(v.dataDict)), + blobEntries = collect(values(v.blobentries)), _version = _getDFGVersion(), ) end -function packVariable( - v::VariableDFG; - includeSolveData::Bool = true, - includeDataEntries::Bool = true, -) - return v -end - -function unpackVariable(variable::VariableDFG; skipVersionCheck::Bool = false) +function unpackVariable(variable; skipVersionCheck::Bool = false) !skipVersionCheck && _versionCheck(variable) # Variable and point type @@ -188,7 +215,7 @@ function unpackVariable(variable::VariableDFG; skipVersionCheck::Bool = false) N = getDimension(variableType) solverDict = Dict{Symbol, State{variableType, pointType, N}}( - map(sd -> sd.solveKey, variable.solverData) .=> + map(sd -> sd.label, variable.solverData) .=> map(sd -> DFG.unpackState(sd), variable.solverData), ) dataDict = Dict{Symbol, Blobentry}( @@ -209,8 +236,3 @@ function unpackVariable(variable::VariableDFG; skipVersionCheck::Bool = false) solvable = variable.solvable, ) end - -VariableCompute(v::VariableCompute) = v -VariableCompute(v::VariableDFG) = unpackVariable(v) -VariableDFG(v::VariableDFG) = v -VariableDFG(v::VariableCompute) = packVariable(v) diff --git a/test/fileDFGTests.jl b/test/fileDFGTests.jl index 96fad868..cbe69bb6 100644 --- a/test/fileDFGTests.jl +++ b/test/fileDFGTests.jl @@ -30,7 +30,7 @@ using UUIDs label = :testing, blobstore = :store, # timestamp = now(localzone()), - timestamp = DFG.ndnow(UTC), + timestamp = DFG.TimeDateZone(now(localzone())), ), ), verts, @@ -43,7 +43,7 @@ using UUIDs label = :testing2, blobstore = :store, # timestamp = ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"), - timestamp = DFG.NanoDate(2014, 5, 30, 21), + timestamp = DFG.TimeDateZone( ZonedDateTime(2014, 5, 30, 21, tz"UTC-4")), ), ), verts, @@ -80,8 +80,8 @@ using UUIDs blobid = uuid4(), label = :testing2, blobstore = :store, - # timestamp = NanoDate(2023, 2, 3, 20, tz"UTC+1"), - timestamp = DFG.NanoDate(2023, 2, 3, 20), + # timestamp = ZonedDateTime(2023, 2, 3, 20, tz"UTC+1"), + timestamp = DFG.TimeDateZone(ZonedDateTime(2023, 2, 3, 20, tz"UTC+1")), ) addGraphBlobentry!(dfg, be) diff --git a/test/iifInterfaceTests.jl b/test/iifInterfaceTests.jl index a5dd1bc6..f56bc412 100644 --- a/test/iifInterfaceTests.jl +++ b/test/iifInterfaceTests.jl @@ -177,8 +177,8 @@ end @test getLabel(v1) == v1.label @test getTags(v1) == v1.tags @test getTimestamp(v1) == v1.timestamp - @test getState(v1, :default) === v1.solverDataDict[:default] - @test refStates(v1) == v1.solverDataDict + @test getState(v1, :default) === v1.states[:default] + @test refStates(v1) == v1.states @test typeof(getVariableType(v1)) == Position{1} @test typeof(getVariableType(v2)) == Position{1} diff --git a/test/interfaceTests.jl b/test/interfaceTests.jl index 46adc894..08cefda3 100644 --- a/test/interfaceTests.jl +++ b/test/interfaceTests.jl @@ -6,7 +6,7 @@ if false using Dates using UUIDs using TimeZones - using NanoDates + using TimesDates include("testBlocks.jl") diff --git a/test/testBlocks.jl b/test/testBlocks.jl index d1c10001..dadddd79 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -274,12 +274,12 @@ function DFGVariableSCA() solverDataDict = Dict(:default => State{TestVariableType1}()), ) - # v1.solverDataDict[:default].val[1] = [0.0;] - # v1.solverDataDict[:default].bw[1] = [1.0;] - # v2.solverDataDict[:default].val[1] = [0.0;0.0] - # v2.solverDataDict[:default].bw[1] = [1.0;1.0] - # v3.solverDataDict[:default].val[1] = [0.0;0.0] - # v3.solverDataDict[:default].bw[1] = [1.0;1.0] + # v1.states[:default].val[1] = [0.0;] + # v1.states[:default].bw[1] = [1.0;] + # v2.states[:default].val[1] = [0.0;0.0] + # v2.states[:default].bw[1] = [1.0;1.0] + # v3.states[:default].val[1] = [0.0;0.0] + # v3.states[:default].bw[1] = [1.0;1.0] @test getLabel(v1) == v1_lbl @test getTags(v1) == v1_tags @@ -290,7 +290,7 @@ function DFGVariableSCA() @test getSolvable(v2) == 1 # TODO direct use is not recommended, use accessors, maybe not export or deprecate - @test refStates(v1) == v1.solverDataDict + @test refStates(v1) == v1.states # @test getMetadata(v1) == Dict{Symbol, MetadataTypes}() @@ -611,7 +611,7 @@ function VSDTestBlock!(fg, v1) @test_throws LabelNotFoundError getState(fg, :a, :parametric) #FIXME copied from lower - @test getState(v1, :default) === v1.solverDataDict[:default] + @test getState(v1, :default) === v1.states[:default] # Add new VND of type ContinuousScalar to :x0 # Could also do State(ContinuousScalar()) @@ -766,7 +766,7 @@ function blobsStoresTestBlock!(fg) origin = "origin2", description = "description2", mimetype = "mimetype2", - timestamp = DFG.NanoDate("2020-08-12T12:00:00.000"), + timestamp = DFG.TimeDateZone("2020-08-12T12:00:00.000Z"), ) de2_update = Blobentry(; blobid = uuid4(), @@ -776,7 +776,7 @@ function blobsStoresTestBlock!(fg) origin = "origin2", description = "description2", mimetype = "mimetype2", - timestamp = DFG.NanoDate("2020-08-12T12:00:00.000"), + timestamp = DFG.TimeDateZone("2020-08-12T12:00:00.000Z"), ) @test getLabel(de1) == de1.label @test getTimestamp(de1) == de1.timestamp @@ -1550,13 +1550,13 @@ function FileDFGTestBlock(testDFGAPI; kwargs...) # vnd.BayesNetVertID = :outid # push!(vnd.BayesNetOutVertIDs, :id) # vnd.bw[1] = [1.0;] - vnd.dontmargin = true + # vnd.dontmargin = true # vnd.eliminated = true - vnd.infoPerCoord .= Float64[1.5;] + vnd.observability .= Float64[1.5;] vnd.initialized = true - vnd.ismargin = true + vnd.marginalized = true push!(vnd.separator, :sep) - vnd.solvedCount = 2 + vnd.solves = 2 # vnd.val[1] = [2.0;] #update mergeVariable!(dfg, v4) From bbf1778b9f5a575417709805ede1a6ee52cb3cdd Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Thu, 13 Nov 2025 11:59:47 +0200 Subject: [PATCH 4/8] fix Blobentries --- src/DataBlobs/entities/BlobEntry.jl | 32 ++++++++++++----------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index cbb849c0..e19dc378 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -127,24 +127,18 @@ end const Blobentries = OrderedDict{Symbol, Blobentry} -#TODO write Blobentries as array -# StructUtils.dictlike(::Type{Blobentries}) = false -# StructUtils.structlike(::Type{Blobentries}) = false -# StructUtils.arraylike(::Type{Blobentries}) = false +function StructUtils.lower(entries::Blobentries) + return map(collect(values(entries))) do (entry) + return StructUtils.lower(entry) + end +end -# function StructUtils.lower(entries::Blobentries) -# return map(collect(values(entries))) do (entry) -# return StructUtils.lower(entry) -# end -# end +function StructUtils.makedict(s::StructUtils.StructStyle, T::Type{Blobentries}, json_vector) + entries = T() + foreach(json_vector) do obj + entry, _ = StructUtils.make(s, Blobentry, obj) + return push!(entries, Symbol(obj.label[]) => entry) + end + return entries, nothing +end -# function StructUtils.lift(s::StructUtils.StructStyle, T::Type{Blobentries}, json_vector) -# entries = T() -# @warn "lifting" s T json_vector -# foreach(json_vector) do obj -# global gobj = obj -# return push!(entries, Symbol(obj.label) => StructUtils.make(s, Blobentry, obj)) -# end -# @warn "lifted Blobentries" entries -# return entries, nothing -# end From 2f99002c0f426e9ad7a052f67a3a0f82f28c0f7c Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Thu, 13 Nov 2025 12:06:43 +0200 Subject: [PATCH 5/8] format --- src/Common.jl | 3 +-- src/DataBlobs/entities/BlobEntry.jl | 1 - src/DataBlobs/services/BlobEntry.jl | 3 ++- src/FileDFG/services/FileDFG.jl | 5 ++-- src/entities/DFGFactor.jl | 4 ++- src/entities/DFGVariable.jl | 42 +++++++++++++---------------- src/services/CompareUtils.jl | 8 +++--- src/services/DFGVariable.jl | 1 - src/services/Serialization.jl | 4 +-- test/fileDFGTests.jl | 2 +- 10 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/Common.jl b/src/Common.jl index 161c1771..06892ee8 100644 --- a/src/Common.jl +++ b/src/Common.jl @@ -142,8 +142,7 @@ end ##============================================================================== ## Validation of session, robot, and user labels. ##============================================================================== -global _invalidIds = - ["GRAPH", "AGENT", "VARIABLE", "FACTOR", "BLOB_ENTRY", "FACTORGRAPH"] +global _invalidIds = ["GRAPH", "AGENT", "VARIABLE", "FACTOR", "BLOB_ENTRY", "FACTORGRAPH"] const global _validLabelRegex::Regex = r"^[a-zA-Z][-\w\.\@]*$" diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index e19dc378..d702f715 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -141,4 +141,3 @@ function StructUtils.makedict(s::StructUtils.StructStyle, T::Type{Blobentries}, end return entries, nothing end - diff --git a/src/DataBlobs/services/BlobEntry.jl b/src/DataBlobs/services/BlobEntry.jl index 9a1a1926..a9cf01c2 100644 --- a/src/DataBlobs/services/BlobEntry.jl +++ b/src/DataBlobs/services/BlobEntry.jl @@ -131,7 +131,8 @@ Should be extended if DFG variable is not returned by reference. Also see: [`getBlobentry`](@ref), [`addBlob!`](@ref), [`mergeBlobentry!`](@ref) """ function addBlobentry!(var::VariableCompute, entry::Blobentry) - haskey(var.blobentries, entry.label) && throw(LabelExistsError("Blobentry", entry.label)) + haskey(var.blobentries, entry.label) && + throw(LabelExistsError("Blobentry", entry.label)) var.blobentries[entry.label] = entry return entry end diff --git a/src/FileDFG/services/FileDFG.jl b/src/FileDFG/services/FileDFG.jl index 7cc9bfe4..500665d0 100644 --- a/src/FileDFG/services/FileDFG.jl +++ b/src/FileDFG/services/FileDFG.jl @@ -232,8 +232,9 @@ function loadDFG(file::AbstractString) catch e if e isa MethodError @warn "Deprecated serialization: Failed to read DFG metadata. Attempting to load using the old format. Error:" e - fgPacked = - GraphsDFGs.PackedGraphsDFG(JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG; style = DFGJSONStyle())) + fgPacked = GraphsDFGs.PackedGraphsDFG( + JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG; style = DFGJSONStyle()), + ) else rethrow(e) end diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index 3ec25715..34804130 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -162,7 +162,9 @@ function FactorDFG( state::FactorState = FactorState(), cache = nothing; tags::Set{Symbol} = Set{Symbol}([:FACTOR]), - timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = TimeDateZone(now(localzone())), + timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = TimeDateZone( + now(localzone()), + ), solvable::Int = 1, bloblets::Bloblets = Bloblets(), blobentries::Blobentries = Blobentries(), diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index d1e41188..347069a8 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -95,7 +95,7 @@ StructUtils.lift(::Type{<:State}, obj) = DFG.unpackState(obj) ##------------------------------------------------------------------------------ ## States - OrderedDict{Symbol, State} -const States = OrderedDict{Symbol, State{T, P, N}} where {T<:AbstractStateType,P,N} +const States = OrderedDict{Symbol, State{T, P, N}} where {T <: AbstractStateType, P, N} StructUtils.dictlike(::Type{<:States}) = false StructUtils.structlike(::Type{<:States}) = false @@ -107,7 +107,11 @@ function StructUtils.lower(states::States) end end -function StructUtils.lift(::StructUtils.StructStyle, S::Type{<:States{T}}, json_vector) where T +function StructUtils.lift( + ::StructUtils.StructStyle, + S::Type{<:States{T}}, + json_vector, +) where {T} states = S() foreach(json_vector) do obj return push!(states, Symbol(obj.label) => StructUtils.make(State{T}, obj)) @@ -170,7 +174,10 @@ version(::Type{<:VariableDFG}) = v"0.29" refStates(v::VariableDFG) = v.states #NOTE fielddefaults and fieldtags not through @kwarg macro due to error with State{T, P, N} -function StructUtils.fielddefaults(::StructUtils.StructStyle, ::Type{VariableDFG{T,P,N}}) where {T,P,N} +function StructUtils.fielddefaults( + ::StructUtils.StructStyle, + ::Type{VariableDFG{T, P, N}}, +) where {T, P, N} return ( timestamp = TimeDateZone(now(localzone())), tags = Set{Symbol}(), @@ -223,8 +230,12 @@ function VariableDFG( timestamp = TimeDateZone(timestamp) end if !isnothing(nanosecondtime) - @assert isnothing(steadytime), "nanosecondtime is replaced by steadytime. Cannot specify both steadytime and nanosecondtime" - Base.depwarn("nanosecondtime kwarg is deprecated, use steadytime instead", :VariableDFG) + @assert isnothing(steadytime), + "nanosecondtime is replaced by steadytime. Cannot specify both steadytime and nanosecondtime" + Base.depwarn( + "nanosecondtime kwarg is deprecated, use steadytime instead", + :VariableDFG, + ) steadytime = Nanosecond(nanosecondtime) end if !isnothing(smalldata) @@ -235,25 +246,13 @@ function VariableDFG( N = getDimension(T) P = getPointType(T) - return VariableDFG{T, P, N}(; - label, - steadytime, - solvable, - tags, - timestamp, - kwargs..., - ) + return VariableDFG{T, P, N}(; label, steadytime, solvable, tags, timestamp, kwargs...) end function VariableDFG(label::Symbol, state::State; kwargs...) - return VariableCompute(; - label, - states = OrderedDict(state.label => state), - kwargs..., - ) + return VariableCompute(; label, states = OrderedDict(state.label => state), kwargs...) end - # Base.getproperty(x::VariableCompute, f::Symbol) = begin # if f == :solvable # getfield(x, f)[] @@ -321,10 +320,7 @@ Base.@kwdef struct VariableSkeleton <: AbstractGraphVariable tags::Set{Symbol} = Set{Symbol}() end -function VariableSkeleton( - label::Symbol, - tags = Set{Symbol}(); -) +function VariableSkeleton(label::Symbol, tags = Set{Symbol}();) return VariableSkeleton(label, tags) end diff --git a/src/services/CompareUtils.jl b/src/services/CompareUtils.jl index f82675b9..8858dca1 100644 --- a/src/services/CompareUtils.jl +++ b/src/services/CompareUtils.jl @@ -213,10 +213,12 @@ function compare(a::State, b::State) !isapprox(a.observability, b.observability; atol = 1e-13) && @debug("infoPerCoord is not equal") === nothing && return false - a.marginalized != b.marginalized && @debug("ismargin is not equal") === nothing && return false + a.marginalized != b.marginalized && + @debug("ismargin is not equal") === nothing && + return false # a.dontmargin != b.dontmargin && - # @debug("dontmargin is not equal") === nothing && - # return false + # @debug("dontmargin is not equal") === nothing && + # return false getVariableType(a) != getVariableType(b) && @debug("variableType is not equal") === nothing && return false diff --git a/src/services/DFGVariable.jl b/src/services/DFGVariable.jl index 8ac47546..96da1c58 100644 --- a/src/services/DFGVariable.jl +++ b/src/services/DFGVariable.jl @@ -671,4 +671,3 @@ function listStates( end return labels end - diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index f562173b..61cc0d0f 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -93,10 +93,10 @@ function packState(d::State{T}) where {T <: StateType} length(d.covar) > 1 && @warn( "Packing of more than one parametric covariance is NOT supported yet, only packing first." ) - + return ( label = d.label, - vecval =_val, + vecval = _val, dimval = size(castval, 1), vecbw = d.bw[:], dimbw = size(d.bw, 1), diff --git a/test/fileDFGTests.jl b/test/fileDFGTests.jl index cbe69bb6..f7735ab6 100644 --- a/test/fileDFGTests.jl +++ b/test/fileDFGTests.jl @@ -43,7 +43,7 @@ using UUIDs label = :testing2, blobstore = :store, # timestamp = ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"), - timestamp = DFG.TimeDateZone( ZonedDateTime(2014, 5, 30, 21, tz"UTC-4")), + timestamp = DFG.TimeDateZone(ZonedDateTime(2014, 5, 30, 21, tz"UTC-4")), ), ), verts, From 45d075a22d344754ca45ef6e7502d1ca00c52bfe Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Thu, 13 Nov 2025 12:12:36 +0200 Subject: [PATCH 6/8] missed this file --- src/serialization/DFGStructStyles.jl | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/serialization/DFGStructStyles.jl diff --git a/src/serialization/DFGStructStyles.jl b/src/serialization/DFGStructStyles.jl new file mode 100644 index 00000000..a5d713a4 --- /dev/null +++ b/src/serialization/DFGStructStyles.jl @@ -0,0 +1,9 @@ +struct DFGJSONStyle <: JSON.JSONStyle end + +StructUtils.structlike(::DFGJSONStyle, ::Type{Base.RefValue{Int}}) = false +StructUtils.lower(::DFGJSONStyle, x::Base.RefValue{Int}) = x[] +StructUtils.lift(::DFGJSONStyle, ::Type{Base.RefValue{Int}}, x) = Ref(x), nothing + +StructUtils.structlike(::DFGJSONStyle, ::Type{TimeDateZone}) = false +StructUtils.lower(::DFGJSONStyle, x::TimeDateZone) = string(x) +StructUtils.lift(::DFGJSONStyle, ::Type{TimeDateZone}, x) = TimeDateZone(x), nothing From d1e16a1310b91c313c4bc448ab90266d86324ce3 Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Thu, 13 Nov 2025 14:55:19 +0200 Subject: [PATCH 7/8] fix some tests --- src/DataBlobs/entities/BlobEntry.jl | 7 +++- src/DistributedFactorGraphs.jl | 5 ++- src/entities/DFGFactor.jl | 12 ++---- src/entities/DFGVariable.jl | 14 +++++-- src/services/AbstractDFG.jl | 8 ++-- src/services/CompareUtils.jl | 2 +- src/services/CustomPrinting.jl | 2 +- src/services/DFGVariable.jl | 8 +--- src/services/Serialization.jl | 58 ----------------------------- test/compareTests.jl | 8 ++-- test/interfaceTests.jl | 11 ++++-- test/testBlocks.jl | 51 +++++++++++++------------ 12 files changed, 66 insertions(+), 120 deletions(-) diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index d702f715..39dd6e08 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -50,7 +50,12 @@ StructUtils.@kwarg struct Blobentry end version(::Type{Blobentry}) = v"0.1.0" -function Blobentry(label::Symbol, blobstore = :default; metadata, kwargs...) +function Blobentry( + label::Symbol, + blobstore = :default; + metadata = JSONText("{}"), + kwargs..., +) if !(metadata isa JSONText) metadata = JSONText(JSON.json(metadata)) end diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 9910c3e5..268ba11a 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -331,6 +331,7 @@ export @defStateType #TODO Should this be exported? # list of unstable functions not exported any more # will move to public or deprecate over time const unstable_functions::Vector{Symbol} = [ + :refStates, #internal maybe make public :InMemoryBlobstore, :MetadataTypes, #maybe make public after metadata stable :getFactorState, # FIXME getFactorState were questioned and being reviewed again for name, other than that they are checked. @@ -396,7 +397,7 @@ const unstable_functions::Vector{Symbol} = [ :plotDFG, :pack, :packDistribution, - :packVariable, + # :packVariable, # :packFactor, :packBlob, :packState, @@ -468,7 +469,7 @@ const unstable_functions::Vector{Symbol} = [ :DFGFactor, :PackedFactor, :Factor, - :AbstractPointParametricEst, + # :AbstractPointParametricEst, # :MeanMaxPPE, # :getPPEMax, # :getPPEMean, diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index 34804130..aa3012c7 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -188,22 +188,16 @@ function FactorDFG( solvercache = Ref(cache) end - # deprecated in v0.29 + # TODO v0.29 deprecate ZonedDateTime or convert internally if timestamp isa ZonedDateTime - Base.depwarn( - "`FactorDFG` timestamp as `ZonedDateTime` is deprecated, use `TimeDateZone(timestamp.utc_datetime)` instead", - :FactorDFG, - ) - nd_timestamp = TimeDateZone(timestamp.utc_datetime) - else - nd_timestamp = timestamp + timestamp = TimeDateZone(timestamp) end return FactorDFG( label, tags, Tuple(variableorder), - nd_timestamp, + timestamp, Ref(solvable), bloblets, observation, diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index 347069a8..c040909c 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -54,7 +54,7 @@ $(TYPEDFIELDS) """ observability::Vector{Float64} = Float64[]#zeros(getDimension(T)) #TODO renamed from infoPerCoord """ - Should this variable solveKey be treated as marginalized in inference computations. + Should this state be treated as marginalized in inference computations. """ marginalized::Bool = false #TODO renamed from ismargin # """ @@ -62,7 +62,7 @@ $(TYPEDFIELDS) # """ # dontmargin::Bool = false """ - How many times has a solver updated this variable solveKey estimte. + How many times has a solver updated this state estimate. """ solves::Int = 0 # TODO renamed from solvedCount # """ @@ -242,6 +242,9 @@ function VariableDFG( Base.depwarn("smalldata kwarg is deprecated, use bloblets instead", :VariableDFG) #TODO convert smalldata to bloblets end + if solvable isa Int + solvable = Ref(solvable) + end union!(tags, [:VARIABLE]) N = getDimension(T) @@ -250,7 +253,12 @@ function VariableDFG( end function VariableDFG(label::Symbol, state::State; kwargs...) - return VariableCompute(; label, states = OrderedDict(state.label => state), kwargs...) + return VariableDFG( + label, + getStateType(state); + states = OrderedDict(state.label => state), + kwargs..., + ) end # Base.getproperty(x::VariableCompute, f::Symbol) = begin diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 7c8e96ee..bb6686f0 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -756,8 +756,8 @@ end Find and return the closest timestamp from two sets of Tuples. Also return the minimum delta-time (`::Millisecond`) and how many elements match from the two sets are separated by the minimum delta-time. """ function findClosestTimestamp( - setA::Vector{Tuple{ZonedDateTime, T}}, - setB::Vector{Tuple{ZonedDateTime, S}}, + setA::Vector{Tuple{TimeDateZone, T}}, + setB::Vector{Tuple{TimeDateZone, S}}, ) where {S, T} # # build matrix of delta times, ranges on rows x vars on columns @@ -798,7 +798,7 @@ ls, listVariables, findClosestTimestamp """ function findVariableNearTimestamp( dfg::AbstractDFG, - timest::ZonedDateTime, + timest::TimeDateZone,#ZonedDateTime, regexFilter::Union{Nothing, Regex} = nothing; tags::Vector{Symbol} = Symbol[], solvable::Int = 0, @@ -848,7 +848,7 @@ function findVariableNearTimestamp( ) return findVariableNearTimestamp( dfg, - ZonedDateTime(timest, timezone), + TimeDateZone(timest, timezone), regexFilter; kwargs..., ) diff --git a/src/services/CompareUtils.jl b/src/services/CompareUtils.jl index 8858dca1..fc535a02 100644 --- a/src/services/CompareUtils.jl +++ b/src/services/CompareUtils.jl @@ -241,7 +241,7 @@ function compareVariable( skiplist = union( [ :attributes; - :solverDataDict; + :states; :createdTimestamp; :lastUpdatedTimestamp; :timezone; diff --git a/src/services/CustomPrinting.jl b/src/services/CustomPrinting.jl index 08a346b6..3a1652fb 100644 --- a/src/services/CustomPrinting.jl +++ b/src/services/CustomPrinting.jl @@ -74,7 +74,7 @@ function printVariable( printstyled(ioc, summary(vert); bold = true, color = :blue) println(ioc, "") - :solver in skipfields && push!(skipfields, :solverDataDict) + :solver in skipfields && push!(skipfields, :states) t = typeof(vert) fields = setdiff(fieldnames(t), skipfields) diff --git a/src/services/DFGVariable.jl b/src/services/DFGVariable.jl index 96da1c58..67c7047e 100644 --- a/src/services/DFGVariable.jl +++ b/src/services/DFGVariable.jl @@ -578,16 +578,10 @@ function copytoState!( stateLabel::Symbol, state::State, ) - newstate = State( - getVariableType(state); - (k => deepcopy(getproperty(state, k)) for k in fieldnames(State))..., - solveKey = stateLabel, - ) + newstate = State(state; label = stateLabel) return mergeState!(dfg, variableLabel, newstate) end -# - """ $(SIGNATURES) Delete the variable `State` by label, returns the number of deleted elements. diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index 61cc0d0f..0348e968 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -178,61 +178,3 @@ function unpackState(d) solves = d.solves, ) end - -##============================================================================== -## Variable Packing and unpacking -##============================================================================== - -function packVariable( - v::VariableCompute; - includeSolveData::Bool = true, - includeDataEntries::Bool = true, -) - return ( - id = v.id, - label = v.label, - timestamp = v.timestamp, - nstime = string(v.nstime.value), - tags = collect(v.tags), # Symbol.() - solverData = packState.(collect(values(v.states))), - metadata = base64encode(JSON.json(v.smallData)), - solvable = getSolvable(v), - variableType = stringVariableType(DFG.getVariableType(v)), - blobEntries = collect(values(v.blobentries)), - _version = _getDFGVersion(), - ) -end - -function unpackVariable(variable; skipVersionCheck::Bool = false) - !skipVersionCheck && _versionCheck(variable) - - # Variable and point type - variableType = parseVariableType(variable.variableType) - isnothing(variableType) && error( - "Cannot deserialize variableType '$(variable.variableType)' in variable '$(variable.label)'", - ) - pointType = DFG.getPointType(variableType) - - N = getDimension(variableType) - solverDict = Dict{Symbol, State{variableType, pointType, N}}( - map(sd -> sd.label, variable.solverData) .=> - map(sd -> DFG.unpackState(sd), variable.solverData), - ) - dataDict = Dict{Symbol, Blobentry}( - map(de -> de.label, variable.blobEntries) .=> variable.blobEntries, - ) - metadata = JSON.parse(base64decode(variable.metadata), Dict{Symbol, DFG.MetadataTypes}) - - return VariableCompute( - variable.label, - variableType; - id = variable.id, - timestamp = variable.timestamp, - nstime = Nanosecond(variable.nstime), - tags = Set(variable.tags), - solverDataDict = solverDict, - smallData = metadata, - dataDict = dataDict, - solvable = variable.solvable, - ) -end diff --git a/test/compareTests.jl b/test/compareTests.jl index 3af8c3ea..fe3419be 100644 --- a/test/compareTests.jl +++ b/test/compareTests.jl @@ -8,9 +8,9 @@ using Dates ## Generated compare functions # State -vnd1 = State(TestVariableType1()) +vnd1 = State(:default, TestVariableType1()) vnd2 = deepcopy(vnd1) -vnd3 = State(TestVariableType2()) +vnd3 = State(:default, TestVariableType2()) @test vnd1 == vnd2 push!(vnd1.val, [1.0;]) @@ -49,9 +49,9 @@ f3 = FactorCompute(:f1, [:b, :a], TestFunctorInferenceType1()) @test !(f1 == f3) ## Compare functions -vnd1 = State(TestVariableType1()) +vnd1 = State(:default, TestVariableType1()) vnd2 = deepcopy(vnd1) -vnd3 = State(TestVariableType2()) +vnd3 = State(:default, TestVariableType2()) @test compare(vnd1, vnd2) @test !compare(vnd1, vnd3) diff --git a/test/interfaceTests.jl b/test/interfaceTests.jl index 08cefda3..5a7426b8 100644 --- a/test/interfaceTests.jl +++ b/test/interfaceTests.jl @@ -22,6 +22,9 @@ if false global_logger(logger) end +# DFG.@usingDFG true +# include("testBlocks.jl") + # DFG Accessors @testset "DFG Structure and Accessors" begin # Constructors @@ -64,13 +67,13 @@ end @test printVariable(iobuf, var1; skipfields = [:timestamp, :solver, :ppe, :nstime]) === nothing - @test String(take!(iobuf)) == - "VariableCompute{TestVariableType1, Vector{Float64}, 1}\nid:\nnothing\nlabel:\n:a\ntags:\nSet([:VARIABLE, :POSE])\nsmallData:\nDict{Symbol, Union{Bool, Float64, Int64, Vector{Bool}, Vector{Float64}, Vector{Int64}, Vector{String}, String}}()\ndataDict:\nDict{Symbol, Blobentry}()\nsolvable:\nRefValue{Int64}(0)\n" - # "VariableCompute{TestVariableType1, Vector{Float64}, 1}\nid:\nnothing\nlabel:\n:a\ntags:\nSet([:VARIABLE, :POSE])\nsmallData:\nDict{Symbol, Union{Bool, Float64, Int64, Vector{Bool}, Vector{Float64}, Vector{Int64}, Vector{String}, String}}(:small=>\"data\")\ndataDict:\nDict{Symbol, Blobentry}()\nsolvable:\n0\n" + varstr = String(take!(iobuf)) + @test occursin(r"VariableDFG", varstr) + @test occursin(r"label", varstr) @test printVariable(iobuf, var1; short = true) === nothing varstr = String(take!(iobuf)) - @test occursin(r"VariableCompute", varstr) + @test occursin(r"VariableDFG", varstr) @test occursin(r"timestamp", varstr) @test occursin(r"label", varstr) @test occursin(r"bandwidths", varstr) diff --git a/test/testBlocks.jl b/test/testBlocks.jl index dadddd79..5d5ad46f 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -257,12 +257,16 @@ function DFGVariableSCA() TestVariableType1(); tags = v1_tags, solvable = 0, - solverDataDict = Dict(:default => State{TestVariableType1}()), + states = Dict(:default => State{TestVariableType1}(; label = :default)), + ) + v2 = VariableCompute( + :b, + State{TestVariableType2}(; label = :default); + tags = Set([:VARIABLE, :LANDMARK]), ) - v2 = VariableCompute(:b, State{TestVariableType2}(); tags = Set([:VARIABLE, :LANDMARK])) v3 = VariableCompute( :c, - State{TestVariableType2}(); + State{TestVariableType2}(; label = :default); timestamp = ZonedDateTime("2020-08-11T00:12:03.000-05:00"), ) @@ -271,7 +275,7 @@ function DFGVariableSCA() TestVariableType1(); tags = v1_tags, solvable = 0, - solverDataDict = Dict(:default => State{TestVariableType1}()), + states = Dict(:default => State{TestVariableType1}(; label = :default)), ) # v1.states[:default].val[1] = [0.0;] @@ -290,7 +294,7 @@ function DFGVariableSCA() @test getSolvable(v2) == 1 # TODO direct use is not recommended, use accessors, maybe not export or deprecate - @test refStates(v1) == v1.states + @test DFG.refStates(v1) == v1.states # @test getMetadata(v1) == Dict{Symbol, MetadataTypes}() @@ -578,7 +582,7 @@ function VSDTestBlock!(fg, v1) # **State** # - `getSolveInProgress` - vnd = State{TestVariableType1}(; solveKey = :parametric) + vnd = State{TestVariableType1}(; label = :parametric) # vnd.val[1] = [0.0;] # vnd.bw[1] = [1.0;] @test addState!(fg, :a, vnd) == vnd @@ -616,7 +620,7 @@ function VSDTestBlock!(fg, v1) # Add new VND of type ContinuousScalar to :x0 # Could also do State(ContinuousScalar()) - vnd = State{TestVariableType1}(; solveKey = :parametric) + vnd = State{TestVariableType1}(; label = :parametric) # vnd.val[1] = [0.0;] # vnd.bw[1] = [1.0;] @@ -636,7 +640,6 @@ function VSDTestBlock!(fg, v1) return nothing - #TODO solverDataDict() not deprecated # @test refStates(newvar) == refStates(v1) # @test @test_deprecated mergeUpdateVariableSolverData!(fg, newvar) @@ -735,17 +738,7 @@ function DataEntriesTestBlock!(fg, v2) #delete from dfg @test deleteBlobentry!(fg, :a, :key2) == 1 @test listBlobentries(v1) == Symbol[] - deleteBlobentry!(fg, :b, :key2) - - # packed variable data entries - pacv = packVariable(v1) - @test addBlobentry!(pacv, de1) == de1 - @test hasBlobentry(pacv, :key1) - @test deepcopy(de1) == getBlobentry(pacv, :key1) - @test getBlobentries(pacv) == [deepcopy(de1)] - @test issetequal(listBlobentries(pacv), [:key1]) - # @test deleteBlobentry!(pacv, de1) == de1 - + return deleteBlobentry!(fg, :b, :key2) end function blobsStoresTestBlock!(fg) @@ -756,7 +749,7 @@ function blobsStoresTestBlock!(fg) crchash = 0xAAAA, origin = "origin1", description = "description1", - mimetype = "mimetype1", + mimetype = MIME("mimetype1"), ) de2 = Blobentry(; blobid = uuid4(), @@ -765,7 +758,7 @@ function blobsStoresTestBlock!(fg) crchash = 0xFFFF, origin = "origin2", description = "description2", - mimetype = "mimetype2", + mimetype = MIME("mimetype2"), timestamp = DFG.TimeDateZone("2020-08-12T12:00:00.000Z"), ) de2_update = Blobentry(; @@ -775,7 +768,7 @@ function blobsStoresTestBlock!(fg) crchash = 0x0123, origin = "origin2", description = "description2", - mimetype = "mimetype2", + mimetype = MIME("mimetype2"), timestamp = DFG.TimeDateZone("2020-08-12T12:00:00.000Z"), ) @test getLabel(de1) == de1.label @@ -1157,9 +1150,15 @@ function connectivityTestGraph( dfg = T(; graphLabel = :testGraph) vars = vcat( - map(n -> VARTYPE(Symbol("x$n"), State{TestVariableType1}()), 1:numNodesType1), map( - n -> VARTYPE(Symbol("x$(numNodesType1+n)"), State{TestVariableType2}()), + n -> VARTYPE(Symbol("x$n"), State{TestVariableType1}(; label = :default)), + 1:numNodesType1, + ), + map( + n -> VARTYPE( + Symbol("x$(numNodesType1+n)"), + State{TestVariableType2}(; label = :default), + ), 1:numNodesType2, ), ) @@ -1391,10 +1390,10 @@ function ProducingDotFiles( dotdfg = testDFGAPI(; graphLabel = :testGraph) if v1 === nothing - v1 = VARTYPE(:a, State{TestVariableType1}()) + v1 = VARTYPE(:a, State{TestVariableType1}(; label = :default)) end if v2 === nothing - v2 = VARTYPE(:b, State{TestVariableType1}()) + v2 = VARTYPE(:b, State{TestVariableType1}(; label = :default)) end if f1 === nothing if (FACTYPE == FactorCompute) From 7b9bb92ad89b3625329c3934603b2b06f8a0a740 Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Fri, 14 Nov 2025 10:23:51 +0200 Subject: [PATCH 8/8] fix more tests --- .github/workflows/ci.yml | 2 +- Project.toml | 2 -- ext/DFGPlots.jl | 3 +-- src/Deprecated.jl | 6 ++++++ src/DistributedFactorGraphs.jl | 6 +++--- src/entities/DFGFactor.jl | 2 +- src/entities/DFGVariable.jl | 18 +++++++++--------- src/serialization/DFGStructStyles.jl | 8 ++++++-- src/services/DFGVariable.jl | 25 +------------------------ test/GraphsDFGSummaryTypes.jl | 9 +++------ test/iifInterfaceTests.jl | 10 ++-------- test/runtests.jl | 10 ++-------- test/testBlocks.jl | 28 ++++++++++------------------ 13 files changed, 45 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a47c3803..e15a7a50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: - name: Run tests env: - IIF_TEST: true + IIF_TEST: false #FIXME skipping until IIF stable again uses: julia-actions/julia-runtest@latest - name: Process Coverage uses: julia-actions/julia-processcoverage@v1 diff --git a/Project.toml b/Project.toml index 82902fd9..a03a4435 100644 --- a/Project.toml +++ b/Project.toml @@ -6,7 +6,6 @@ version = "0.28.0" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" -Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" @@ -45,7 +44,6 @@ Arrow = "2.7" Base64 = "1.10" CSV = "0.10" CodecZlib = "0.7" -Colors = "0.10, 0.11, 0.12, 0.13" Dates = "1.10" Distributions = "0.23, 0.24, 0.25" DocStringExtensions = "0.8, 0.9" diff --git a/ext/DFGPlots.jl b/ext/DFGPlots.jl index 99681759..753098b3 100644 --- a/ext/DFGPlots.jl +++ b/ext/DFGPlots.jl @@ -1,10 +1,9 @@ module DFGPlots -using Colors using Graphs using DocStringExtensions using GraphMakie - +using GraphMakie.Makie: @colorant_str, RGB using DistributedFactorGraphs import DistributedFactorGraphs: plotDFG diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 783a9a5c..230015fc 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -151,6 +151,12 @@ end #TODO is Type correct @deprecate getVariableType(args...) getStateType(args...) + +function getVariableTypeName(v::VariableSummary) + Base.depwarn("getVariableTypeName is deprecated.", :getVariableTypeName) + return v.statetype +end + ## ================================================================================ ## Deprecated in v0.28 ##================================================================================= diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 268ba11a..13a81798 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -359,8 +359,6 @@ const unstable_functions::Vector{Symbol} = [ :findFactorsBetweenNaive, :getAgentLabel, :getGraphLabel, - :getVariableTypeName, - :getVariableType, :getDescription, :getAddHistory, :getSolverParams, @@ -403,7 +401,7 @@ const unstable_functions::Vector{Symbol} = [ :packState, :unpack, :unpackDistribution, - :unpackVariable, + # :unpackVariable, # :unpackFactor, :unpackBlob, :unpackState, @@ -429,6 +427,8 @@ const unstable_functions::Vector{Symbol} = [ # no set on these #deprecated in v0.29 + :getVariableTypeName, + :getVariableType, :setTimestamp, :setMetadata!, # no set, use add merge :setAgentMetadata!, diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index aa3012c7..54099ad3 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -80,7 +80,7 @@ StructUtils.@kwarg struct FactorDFG{T <: AbstractObservation, N} <: AbstractGrap # nstime::Nanosecond #NOTE v0.29 REMOVED as not used, add when needed, or now as steadytime. """Solvable flag for the factor. Accessors: [`getSolvable`](@ref), [`setSolvable!`](@ref)""" - solvable::Base.RefValue{Int} = Ref(1) #& (lower = getindex, lift = Ref) + solvable::Base.RefValue{Int} = Ref{Int}(1) #& (lower = getindex, lift = Ref) """Dictionary of small data associated with this variable. Accessors: [`getBloblet`](@ref), [`addBloblet!`](@ref)""" bloblets::Bloblets = Bloblets() #NOTE v0.29 changed from smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}() diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index c040909c..0e93e255 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -110,7 +110,7 @@ end function StructUtils.lift( ::StructUtils.StructStyle, S::Type{<:States{T}}, - json_vector, + json_vector::Vector, ) where {T} states = S() foreach(json_vector) do obj @@ -165,7 +165,7 @@ $(TYPEDFIELDS) blobentries::Blobentries = Blobentries() #NOTE renamed from dataDict in v0.29 """Solvable flag for the variable. Accessors: [`getSolvable`](@ref), [`setSolvable!`](@ref)""" - solvable::Base.RefValue{Int} = Ref(1) #& (lower = getindex,) + solvable::Base.RefValue{Int} = Ref{Int}(1) #& (lower = getindex,) statetype::Symbol = Symbol(stringVariableType(T())) # TODO autotype or version and statetype _autotype::Nothing = nothing #& (name = :type, lower = _ -> TypeMetadata(VariableDFG)) @@ -216,10 +216,10 @@ The default VariableDFG constructor. #IIF like contruction helper for VariableDFG function VariableDFG( label::Symbol, - stateType::Union{T, Type{T}}; + statetype::Union{T, Type{T}}; tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}(), timestamp::Union{TimeDateZone, ZonedDateTime} = TimeDateZone(now(localzone())), - solvable::Union{Int, Base.RefValue{Int}} = Ref(1), + solvable::Union{Int, Base.RefValue{Int}} = Ref{Int}(1), steadytime::Union{Nothing, Nanosecond} = nothing, nanosecondtime = nothing, smalldata = nothing, @@ -299,9 +299,9 @@ $(TYPEDFIELDS) """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`getTags`](@ref), [`mergeTags!`](@ref), and [`removeTags!`](@ref)""" tags::Set{Symbol} - """Symbol for the variableType for the underlying variable. - Accessor: [`getVariableType`](@ref)""" - variableTypeName::Symbol & (json = (name = "variableType",)) # TODO check from StructTypes.names(::Type{VariableSummary}) = ((:variableTypeName, :variableType),) + """Symbol for the state type for the underlying variable. + Accessor: [`getStateType`](@ref)""" + statetype::Symbol """Dictionary of large data associated with this variable. Accessors: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`mergeBlobentry!`](@ref), and [`deleteBlobentry!`](@ref)""" blobentries::Blobentries @@ -336,12 +336,12 @@ end ## Conversion constructors ##============================================================================== -function VariableSummary(v::VariableCompute) +function VariableSummary(v::VariableCompute{T}) where {T} return VariableSummary( v.label, v.timestamp, copy(v.tags), - Symbol(typeof(getVariableType(v))), + Symbol(stringVariableType(T())), copy(v.blobentries), ) end diff --git a/src/serialization/DFGStructStyles.jl b/src/serialization/DFGStructStyles.jl index a5d713a4..4959ba79 100644 --- a/src/serialization/DFGStructStyles.jl +++ b/src/serialization/DFGStructStyles.jl @@ -2,8 +2,12 @@ struct DFGJSONStyle <: JSON.JSONStyle end StructUtils.structlike(::DFGJSONStyle, ::Type{Base.RefValue{Int}}) = false StructUtils.lower(::DFGJSONStyle, x::Base.RefValue{Int}) = x[] -StructUtils.lift(::DFGJSONStyle, ::Type{Base.RefValue{Int}}, x) = Ref(x), nothing +function StructUtils.lift(::DFGJSONStyle, ::Type{Base.RefValue{Int}}, x::Integer) + return Ref{Int}(x), nothing +end StructUtils.structlike(::DFGJSONStyle, ::Type{TimeDateZone}) = false StructUtils.lower(::DFGJSONStyle, x::TimeDateZone) = string(x) -StructUtils.lift(::DFGJSONStyle, ::Type{TimeDateZone}, x) = TimeDateZone(x), nothing +function StructUtils.lift(::DFGJSONStyle, ::Type{TimeDateZone}, x::AbstractString) + return TimeDateZone(x), nothing +end diff --git a/src/services/DFGVariable.jl b/src/services/DFGVariable.jl index 67c7047e..8dee71d7 100644 --- a/src/services/DFGVariable.jl +++ b/src/services/DFGVariable.jl @@ -436,33 +436,10 @@ function emptyMetadata!(dfg::AbstractDFG, label::Symbol) end ##------------------------------------------------------------------------------ -## Data Entries and Blobs +## Blobentries and Blobs ##------------------------------------------------------------------------------ - ## see DataEntryBlob Folder -##------------------------------------------------------------------------------ -## variableTypeName -##------------------------------------------------------------------------------ -## getter in VariableSummary only -## can be utility function for others -## TODO this should return the variableType object, or try to. it should be getVariableTypeName for the accessor -## TODO Consider parameter N in variableType for dims, and storing constructor in variableTypeName -## TODO or just not having this function at all -# getVariableType(v::VariableSummary) = v.softypename() -##------------------------------------------------------------------------------ - -""" - $SIGNATURES -Retrieve the soft type name symbol for a VariableSummary. ie :Point2, Pose2, etc. -""" -getVariableTypeName(v::VariableSummary) = v.variableTypeName::Symbol - -function getVariableType(v::VariableSummary) - @warn "Looking for type in `Main`. Only use if `variableType` has only one implementation, ie. Pose2. Otherwise use the full variable." - return getfield(Main, v.variableTypeName)() -end - ##============================================================================== ## Layer 2 CRUD and SET ##============================================================================== diff --git a/test/GraphsDFGSummaryTypes.jl b/test/GraphsDFGSummaryTypes.jl index 1b8e3c9b..d8b9cc46 100644 --- a/test/GraphsDFGSummaryTypes.jl +++ b/test/GraphsDFGSummaryTypes.jl @@ -5,9 +5,8 @@ dfg = GraphsDFG{NoSolverParams, VARTYPE, FACTYPE}() function DistributedFactorGraphs.VariableSummary(label::Symbol) return VariableSummary( - nothing, label, - DistributedFactorGraphs.now(localzone()), + TimeDateZone("2025-11-13T15:21:57.474125421+01:00"), Set{Symbol}(), :Pose2, Dict{Symbol, Blobentry}(), @@ -16,9 +15,8 @@ end function DistributedFactorGraphs.VariableSummary(label::Symbol, ::State{T}) where {T} return VariableSummary( - nothing, label, - DistributedFactorGraphs.now(localzone()), + TimeDateZone("2025-11-13T15:21:57.474125421+01:00"), Set{Symbol}(), Symbol(T), Dict{Symbol, Blobentry}(), @@ -30,7 +28,7 @@ function DistributedFactorGraphs.VariableSkeleton(label::Symbol, args...) end function DistributedFactorGraphs.VariableSkeleton(label::Symbol, ::State{T}) where {T} - return VariableSkeleton(nothing, label, Set{Symbol}()) + return VariableSkeleton(label, Set{Symbol}()) end dfg = GraphsDFG{NoSolverParams, VARTYPE, FACTYPE}() @@ -105,7 +103,6 @@ end if VARTYPE == VariableSummary @test getTimestamp(v1) == v1.timestamp - @test getVariableTypeName(v1) == :Pose2 end end diff --git a/test/iifInterfaceTests.jl b/test/iifInterfaceTests.jl index f56bc412..9cd586c3 100644 --- a/test/iifInterfaceTests.jl +++ b/test/iifInterfaceTests.jl @@ -404,14 +404,8 @@ end # Check all fields are equal for all variables for v in ls(summaryGraph) for field in variableFields - if field != :variableTypeName - @test getproperty(getVariable(dfg, v), field) == - getfield(getVariable(summaryGraph, v), field) - else - # Special case to check the symbol variableType is equal to the full variableType. - @test Symbol(typeof(getVariableType(getVariable(dfg, v)))) == - getVariableTypeName(getVariable(summaryGraph, v)) - end + @test getproperty(getVariable(dfg, v), field) == + getfield(getVariable(summaryGraph, v), field) end end for f in lsf(summaryGraph) diff --git a/test/runtests.jl b/test/runtests.jl index 6a7adf04..d5427648 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ using DistributedFactorGraphs using Pkg using Dates using TimeZones +using TimesDates using SHA using UUIDs using Aqua @@ -182,13 +183,6 @@ DFG.@usingDFG true end @testset "Testing Code Quality with Aqua" begin - Aqua.test_ambiguities([DistributedFactorGraphs]) - Aqua.test_unbound_args(DistributedFactorGraphs) - Aqua.test_undefined_exports(DistributedFactorGraphs) - Aqua.test_piracies(DistributedFactorGraphs) - Aqua.test_project_extras(DistributedFactorGraphs) - Aqua.test_stale_deps(DistributedFactorGraphs; ignore = [:Colors]) - Aqua.test_deps_compat(DistributedFactorGraphs) - # Aqua.test_project_toml_formatting(DistributedFactorGraphs) # deprecated in Aqua.jl v0.8 + Aqua.test_all(DistributedFactorGraphs) end end diff --git a/test/testBlocks.jl b/test/testBlocks.jl index 5d5ad46f..f5775f80 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -35,15 +35,15 @@ TestFunctorInferenceType1() = TestFunctorInferenceType1(TestBelief()) TestFunctorInferenceType2() = TestFunctorInferenceType2(TestBelief()) TestAbstractPrior() = TestAbstractPrior(TestBelief()) -struct PackedNothingDistribution <: AbstractPackedBelief - _type::Symbol - function PackedNothingDistribution(; _type::String = "PackedNothingDistribution") - return new(Symbol(_type)) - end -end +# struct PackedNothingDistribution <: AbstractPackedBelief +# _type::Symbol +# function PackedNothingDistribution(; _type::String = "PackedNothingDistribution") +# return new(Symbol(_type)) +# end +# end -DFG.packDistribution(::Nothing) = PackedNothingDistribution() -DFG.unpackDistribution(::PackedNothingDistribution) = nothing +# DFG.packDistribution(::Nothing) = PackedNothingDistribution() +# DFG.unpackDistribution(::PackedNothingDistribution) = nothing struct TestCCW{T <: AbstractObservation} <: FactorCache usrfnc!::T @@ -1357,16 +1357,8 @@ function Summaries(testDFGAPI) # Check all fields are equal for all variables for v in ls(summaryGraph) for field in variableFields - if field != :variableTypeName - @test getproperty(getVariable(dfg, v), field) == - getproperty(getVariable(summaryGraph, v), field) - else - # Special case to check the symbol variableType is equal to the full variableType. - @test Symbol(typeof(getVariableType(getVariable(dfg, v)))) == - getVariableTypeName(getVariable(summaryGraph, v)) - @test getVariableType(getVariable(dfg, v)) == - getVariableType(getVariable(summaryGraph, v)) - end + @test getproperty(getVariable(dfg, v), field) == + getproperty(getVariable(summaryGraph, v), field) end end for f in lsf(summaryGraph)