diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a47c3803..e15a7a50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: - name: Run tests env: - IIF_TEST: true + IIF_TEST: false #FIXME skipping until IIF stable again uses: julia-actions/julia-runtest@latest - name: Process Coverage uses: julia-actions/julia-processcoverage@v1 diff --git a/Project.toml b/Project.toml index f5e849ab..a03a4435 100644 --- a/Project.toml +++ b/Project.toml @@ -6,7 +6,6 @@ version = "0.28.0" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" -Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" @@ -16,7 +15,6 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb" -NanoDates = "46f1a544-deae-4307-8689-c12aa3c955c6" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" @@ -29,6 +27,7 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b" TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" +TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [weakdeps] @@ -45,7 +44,6 @@ Arrow = "2.7" Base64 = "1.10" CSV = "0.10" CodecZlib = "0.7" -Colors = "0.10, 0.11, 0.12, 0.13" Dates = "1.10" Distributions = "0.23, 0.24, 0.25" DocStringExtensions = "0.8, 0.9" @@ -57,7 +55,6 @@ JSON = "1.0.0" LieGroups = "0.1" LinearAlgebra = "1.10" ManifoldsBase = "1, 2" -NanoDates = "1.0.3" OrderedCollections = "1.4" Pkg = "1.4, 1.5" ProgressMeter = "1" @@ -72,6 +69,7 @@ Tar = "1.9" TensorCast = "0.3.3, 0.4" Test = "1.10" TimeZones = "1.3.1" +TimesDates = "0.3.3" UUIDs = "1.10" julia = "1.10" diff --git a/ext/DFGPlots.jl b/ext/DFGPlots.jl index 99681759..753098b3 100644 --- a/ext/DFGPlots.jl +++ b/ext/DFGPlots.jl @@ -1,10 +1,9 @@ module DFGPlots -using Colors using Graphs using DocStringExtensions using GraphMakie - +using GraphMakie.Makie: @colorant_str, RGB using DistributedFactorGraphs import DistributedFactorGraphs: plotDFG diff --git a/src/Common.jl b/src/Common.jl index 8bd1ade9..06892ee8 100644 --- a/src/Common.jl +++ b/src/Common.jl @@ -142,8 +142,7 @@ end ##============================================================================== ## Validation of session, robot, and user labels. ##============================================================================== -global _invalidIds = - ["GRAPH", "AGENT", "VARIABLE", "FACTOR", "PPE", "BLOB_ENTRY", "FACTORGRAPH"] +global _invalidIds = ["GRAPH", "AGENT", "VARIABLE", "FACTOR", "BLOB_ENTRY", "FACTORGRAPH"] const global _validLabelRegex::Regex = r"^[a-zA-Z][-\w\.\@]*$" diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index 20df8da4..39dd6e08 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -40,18 +40,26 @@ StructUtils.@kwarg struct Blobentry """ Additional information that can help a different user of the Blob. """ description::String = "" """ MIME description describing the format of binary data in the `Blob`, e.g. 'image/png' or 'application/json'. """ - mimetype::String = "application/octet-stream" #FIXME ::MIME = MIME("application/octet-stream") + mimetype::MIME = MIME("application/octet-stream") """ Storage for a couple of bytes directly in the graph. Use with caution and keep it small and simple.""" metadata::JSONText = JSONText("{}") """ When the Blob itself was first created. Serialized as an ISO 8601 string.""" - timestamp::NanoDate = ndnow(UTC) & (json = (lower = timestamp,),) + timestamp::TimeDateZone = TimeDateZone(now(localzone())) """ Type version of this Blobentry.""" - version::VersionNumber = version(Blobentry) + version::VersionNumber = DFG.version(Blobentry) end version(::Type{Blobentry}) = v"0.1.0" -function Blobentry(label::Symbol, blobstore = :default; kwargs...) - return Blobentry(; label, blobstore, kwargs...) +function Blobentry( + label::Symbol, + blobstore = :default; + metadata = JSONText("{}"), + kwargs..., +) + if !(metadata isa JSONText) + metadata = JSONText(JSON.json(metadata)) + end + return Blobentry(; label, blobstore, metadata, kwargs...) end # construction helper from existing Blobentry for user overriding via kwargs function Blobentry( @@ -123,3 +131,18 @@ function Base.setproperty!(x::Blobentry, f::Symbol, val) end const Blobentries = OrderedDict{Symbol, Blobentry} + +function StructUtils.lower(entries::Blobentries) + return map(collect(values(entries))) do (entry) + return StructUtils.lower(entry) + end +end + +function StructUtils.makedict(s::StructUtils.StructStyle, T::Type{Blobentries}, json_vector) + entries = T() + foreach(json_vector) do obj + entry, _ = StructUtils.make(s, Blobentry, obj) + return push!(entries, Symbol(obj.label[]) => entry) + end + return entries, nothing +end diff --git a/src/DataBlobs/services/BlobEntry.jl b/src/DataBlobs/services/BlobEntry.jl index b97b003f..a9cf01c2 100644 --- a/src/DataBlobs/services/BlobEntry.jl +++ b/src/DataBlobs/services/BlobEntry.jl @@ -67,17 +67,17 @@ Also see: [`addBlobentry!`](@ref), [`getBlob`](@ref), [`listBlobentries`](@ref) """ function getBlobentry(var::AbstractGraphVariable, key::Symbol) if !hasBlobentry(var, key) - throw(LabelNotFoundError("Blobentry", key, collect(keys(var.dataDict)))) + throw(LabelNotFoundError("Blobentry", key, collect(keys(var.blobentries)))) end - return var.dataDict[key] + return var.blobentries[key] end -function getBlobentry(var::VariableDFG, key::Symbol) - if !hasBlobentry(var, key) - throw(LabelNotFoundError("Blobentry", key)) - end - return var.blobEntries[findfirst(x -> x.label == key, var.blobEntries)] -end +# function getBlobentry(var::VariableDFG, key::Symbol) +# if !hasBlobentry(var, key) +# throw(LabelNotFoundError("Blobentry", key)) +# end +# return var.blobEntries[findfirst(x -> x.label == key, var.blobEntries)] +# end """ $(SIGNATURES) @@ -131,18 +131,19 @@ Should be extended if DFG variable is not returned by reference. Also see: [`getBlobentry`](@ref), [`addBlob!`](@ref), [`mergeBlobentry!`](@ref) """ function addBlobentry!(var::VariableCompute, entry::Blobentry) - haskey(var.dataDict, entry.label) && throw(LabelExistsError("Blobentry", entry.label)) - var.dataDict[entry.label] = entry - return entry -end - -function addBlobentry!(var::VariableDFG, entry::Blobentry) - entry.label in getproperty.(var.blobEntries, :label) && + haskey(var.blobentries, entry.label) && throw(LabelExistsError("Blobentry", entry.label)) - push!(var.blobEntries, entry) + var.blobentries[entry.label] = entry return entry end +# function addBlobentry!(var::VariableDFG, entry::Blobentry) +# entry.label in getproperty.(var.blobEntries, :label) && +# throw(LabelExistsError("Blobentry", entry.label)) +# push!(var.blobEntries, entry) +# return entry +# end + function addBlobentry!(dfg::AbstractDFG, vLbl::Symbol, entry::Blobentry) return addBlobentry!(getVariable(dfg, vLbl), entry) end @@ -158,10 +159,10 @@ If the Blobentry does not exist, it will be added. Notes: """ function mergeBlobentry!(var::AbstractGraphVariable, bde::Blobentry) - if !haskey(var.dataDict, bde.label) + if !haskey(var.blobentries, bde.label) addBlobentry!(var, bde) else - var.dataDict[bde.label] = bde + var.blobentries[bde.label] = bde end return 1 end @@ -179,15 +180,15 @@ Notes: """ function deleteBlobentry!(var::VariableCompute, key::Symbol) !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key)) - delete!(var.dataDict, key) + delete!(var.blobentries, key) return 1 end -function deleteBlobentry!(var::VariableDFG, key::Symbol) - !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key)) - deleteat!(var.blobEntries, findfirst(x -> x.label == key, var.blobEntries)) - return 1 -end +# function deleteBlobentry!(var::VariableDFG, key::Symbol) +# !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key)) +# deleteat!(var.blobEntries, findfirst(x -> x.label == key, var.blobEntries)) +# return 1 +# end function deleteBlobentry!(dfg::AbstractDFG, label::Symbol, key::Symbol) return deleteBlobentry!(getVariable(dfg, label), key) @@ -223,11 +224,11 @@ end Does a blob entry exist with `blobLabel`. """ -hasBlobentry(v::VariableCompute, blobLabel::Symbol) = haskey(v.dataDict, blobLabel) +hasBlobentry(v::VariableCompute, blobLabel::Symbol) = haskey(v.blobentries, blobLabel) -function hasBlobentry(v::VariableDFG, label::Symbol) - return label in getproperty.(v.blobEntries, :label) -end +# function hasBlobentry(v::VariableDFG, label::Symbol) +# return label in getproperty.(v.blobEntries, :label) +# end """ $(SIGNATURES) @@ -235,12 +236,12 @@ end Get blob entries, returns a `Vector{Blobentry}`. """ function getBlobentries(v::VariableCompute) - return collect(values(v.dataDict)) + return collect(values(v.blobentries)) end -function getBlobentries(v::VariableDFG) - return copy(v.blobEntries) -end +# function getBlobentries(v::VariableDFG) +# return copy(v.blobEntries) +# end function getBlobentries( v::AbstractGraphVariable; @@ -289,12 +290,12 @@ const collectBlobentries = gatherBlobentries List the blob entries associated with a particular variable. """ function listBlobentries(var::AbstractGraphVariable) - return collect(keys(var.dataDict)) + return collect(keys(var.blobentries)) end -function listBlobentries(var::VariableDFG) - return getproperty.(var.blobEntries, :label) -end +# function listBlobentries(var::VariableDFG) +# return getproperty.(var.blobEntries, :label) +# end function listBlobentries(dfg::AbstractDFG, label::Symbol) return listBlobentries(getVariable(dfg, label)) diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 355d46ae..230015fc 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -52,8 +52,8 @@ function getSolveInProgress( ) # Variable if var isa VariableCompute - if haskey(getSolverDataDict(var), solveKey) - return getSolverDataDict(var)[solveKey].solveInProgress + if haskey(refStates(var), solveKey) + return refStates(var)[solveKey].solveInProgress else return 0 end @@ -148,6 +148,15 @@ function _getDuplicatedEmptyDFG( # DFG.setDescription!(newDfg, "(Copy of) $(DFG.getDescription(dfg))") return newDfg end + +#TODO is Type correct +@deprecate getVariableType(args...) getStateType(args...) + +function getVariableTypeName(v::VariableSummary) + Base.depwarn("getVariableTypeName is deprecated.", :getVariableTypeName) + return v.statetype +end + ## ================================================================================ ## Deprecated in v0.28 ##================================================================================= @@ -190,7 +199,7 @@ function setSolverData!(v::VariableCompute, data::State, key::Symbol = :default) :setSolverData!, ) @assert key == data.solveKey "State.solveKey=:$(data.solveKey) does not match requested :$(key)" - return v.solverDataDict[key] = data + return v.states[key] = data end @deprecate mergeVariableSolverData!(args...; kwargs...) mergeState!(args...; kwargs...) @@ -289,7 +298,7 @@ end # Related -# [`listSolveKeys`](@ref), [`getSolverDataDict`](@ref), [`listVariables`](@ref) +# [`listSolveKeys`](@ref), [`refStates`](@ref), [`listVariables`](@ref) # """ function listSolveKeys( variable::VariableCompute, @@ -298,7 +307,7 @@ function listSolveKeys( ) Base.depwarn("listSolveKeys is deprecated, use listStates instead.", :listSolveKeys) # - for ky in keys(getSolverDataDict(variable)) + for ky in keys(refStates(variable)) push!(skeys, ky) end @@ -328,7 +337,7 @@ function listSolveKeys( # skeys = Set{Symbol}() varList = listVariables(dfg, filterVariables; tags = tags, solvable = solvable) - for vs in varList #, ky in keys(getSolverDataDict(getVariable(dfg, vs))) + for vs in varList #, ky in keys(refStates(getVariable(dfg, vs))) listSolveKeys(dfg, vs, filterSolveKeys, skeys) end diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 85cc8886..13a81798 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -20,7 +20,7 @@ using DocStringExtensions using Dates using Random using TimeZones -using NanoDates +using TimesDates using JSON export StructUtils # export for use in macros using LinearAlgebra @@ -83,7 +83,7 @@ export FactorDFG, FactorSummary, FactorSkeleton export Blobentry -export State, PackedState +export State ##------------------------------------------------------------------------------ ## Functions @@ -331,6 +331,7 @@ export @defStateType #TODO Should this be exported? # list of unstable functions not exported any more # will move to public or deprecate over time const unstable_functions::Vector{Symbol} = [ + :refStates, #internal maybe make public :InMemoryBlobstore, :MetadataTypes, #maybe make public after metadata stable :getFactorState, # FIXME getFactorState were questioned and being reviewed again for name, other than that they are checked. @@ -358,8 +359,6 @@ const unstable_functions::Vector{Symbol} = [ :findFactorsBetweenNaive, :getAgentLabel, :getGraphLabel, - :getVariableTypeName, - :getVariableType, :getDescription, :getAddHistory, :getSolverParams, @@ -379,7 +378,6 @@ const unstable_functions::Vector{Symbol} = [ :getVariableLabelNumber,# TODO somewhat used, do we deprecate? :getfirstBlobentry,# TODO somewhat used, do we deprecate? :getSolveInProgress,#TODO unused, do we deprecate? - :getSolverDataDict,## TODO deprecated or obsolete :hasTagsNeighbors, :isVariable, :isFactor, @@ -397,13 +395,13 @@ const unstable_functions::Vector{Symbol} = [ :plotDFG, :pack, :packDistribution, - :packVariable, + # :packVariable, # :packFactor, :packBlob, :packState, :unpack, :unpackDistribution, - :unpackVariable, + # :unpackVariable, # :unpackFactor, :unpackBlob, :unpackState, @@ -429,10 +427,13 @@ const unstable_functions::Vector{Symbol} = [ # no set on these #deprecated in v0.29 + :getVariableTypeName, + :getVariableType, :setTimestamp, :setMetadata!, # no set, use add merge :setAgentMetadata!, :setGraphMetadata!, + # :getSolverDataDict,# obsolete #Deprecated in v0.28 :AbstractRelativeMinimize, @@ -468,25 +469,25 @@ const unstable_functions::Vector{Symbol} = [ :DFGFactor, :PackedFactor, :Factor, - :AbstractPointParametricEst, - :MeanMaxPPE, - :getPPEMax, - :getPPEMean, - :getPPESuggested, - :getLastUpdatedTimestamp, - :getPPEDict, - :getVariablePPEDict, - :getVariablePPE, + # :AbstractPointParametricEst, + # :MeanMaxPPE, + # :getPPEMax, + # :getPPEMean, + # :getPPESuggested, + # :getLastUpdatedTimestamp, + # :getPPEDict, + # :getVariablePPEDict, + # :getVariablePPE, :listSolveKeys, :listSupersolves, - :getPPE, - :getPPEs, - :getVariablePPE, - :addPPE!, - :updatePPE!, - :deletePPE!, - :listPPEs, - :mergePPEs!, + # :getPPE, + # :getPPEs, + # :getVariablePPE, + # :addPPE!, + # :updatePPE!, + # :deletePPE!, + # :listPPEs, + # :mergePPEs!, Symbol("@defVariable"), :SmallDataTypes, :NoSolverParams, @@ -531,6 +532,7 @@ include("entities/Bloblet.jl") include("DataBlobs/entities/BlobEntry.jl") include("DataBlobs/entities/BlobStores.jl") +include("serialization/DFGStructStyles.jl") include("serialization/PackedSerialization.jl") include("serialization/DistributionSerialization.jl") diff --git a/src/FileDFG/services/FileDFG.jl b/src/FileDFG/services/FileDFG.jl index f66c215b..500665d0 100644 --- a/src/FileDFG/services/FileDFG.jl +++ b/src/FileDFG/services/FileDFG.jl @@ -40,19 +40,19 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG; saveMetadata::Bool = map(f -> rm("$factorFolder/$f"), readdir(factorFolder)) # Variables @showprogress "saving variables" for v in variables - vPacked = packVariable(v) - JSON.json("$varFolder/$(v.label).json", vPacked) + # vPacked = packVariable(v) + JSON.json("$varFolder/$(v.label).json", v; style = DFGJSONStyle()) end # Factors @showprogress "saving factors" for f in factors - JSON.json("$factorFolder/$(f.label).json", f) + JSON.json("$factorFolder/$(f.label).json", f; style = DFGJSONStyle()) end #GraphsDFG metadata if saveMetadata @assert isa(dfg, GraphsDFG) "only metadata for GraphsDFG are supported" @info "saving dfg metadata" fgPacked = GraphsDFGs.packDFGMetadata(dfg) - JSON.json("$savepath/dfg.json", fgPacked) + JSON.json("$savepath/dfg.json", fgPacked; style = DFGJSONStyle()) end savedir = dirname(savepath) # is this a path of just local name? #344 -- workaround with unique names @@ -138,7 +138,7 @@ function loadDFG!( @assert isa(dfgLoadInto, GraphsDFG) "Only GraphsDFG metadata are supported" @info "loading dfg metadata" jstr = read("$folder/dfg.json", String) - fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG) + fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG; style = DFGJSONStyle()) GraphsDFGs.unpackDFGMetadata!(dfgLoadInto, fgPacked) end @@ -164,7 +164,7 @@ function loadDFG!( # type instability on `variables` as either `::Vector{Variable}` or `::Vector{VariableCompute{<:}}` (vector of abstract) variables = @showprogress 1 "loading variables" asyncmap(varFiles) do varFile jstr = read("$varFolder/$varFile", String) - packedvar = JSON.parse(jstr, VariableDFG) + packedvar = JSON.parse(jstr, VariableDFG; style = DFGJSONStyle()) v = usePackedVariable ? packedvar : unpackVariable(packedvar) return addVariable!(dfgLoadInto, v) end @@ -176,7 +176,7 @@ function loadDFG!( # `factors` is not type stable `::Vector{Factor}` or `::Vector{FactorCompute{<:}}` (vector of abstract) factors = @showprogress 1 "loading factors" asyncmap(factorFiles) do factorFile - f = JSON.parsefile("$factorFolder/$factorFile", FactorDFG) + f = JSON.parsefile("$factorFolder/$factorFile", FactorDFG; style = DFGJSONStyle()) return addFactor!(dfgLoadInto, f) end @@ -228,12 +228,13 @@ function loadDFG(file::AbstractString) #TODO deprecate old format, v0.28 local fgPacked try - fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG) + fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG; style = DFGJSONStyle()) catch e if e isa MethodError @warn "Deprecated serialization: Failed to read DFG metadata. Attempting to load using the old format. Error:" e - fgPacked = - GraphsDFGs.PackedGraphsDFG(JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG)) + fgPacked = GraphsDFGs.PackedGraphsDFG( + JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG; style = DFGJSONStyle()), + ) else rethrow(e) end diff --git a/src/entities/Bloblet.jl b/src/entities/Bloblet.jl index ff724f17..2eec982c 100644 --- a/src/entities/Bloblet.jl +++ b/src/entities/Bloblet.jl @@ -21,3 +21,46 @@ function StructUtils.lift(::Type{Bloblets}, json_vector::Vector) Symbol(x["label"]) => Bloblet(Symbol(x["label"]), x["val"]) for x in json_vector ) end + +""" + $(SIGNATURES) +""" +function getBloblet(node, label::Symbol) + !haskey(refBloblets(node), label) && throw(LabelNotFoundError("Bloblet", label)) + return refBloblets(node)[label] +end + +""" + $(SIGNATURES) +""" +function addBloblet!(node, bloblet::Bloblet) + label = getLabel(bloblet) + haskey(refBloblets(node), label) && throw(LabelExistsError("Bloblet", label)) + refBloblets(node)[label] = bloblet + return bloblet +end + +""" + $(SIGNATURES) +""" +function mergeBloblet!(node, bloblet::Bloblet) + refBloblets(node)[getLabel(bloblet)] = bloblet + return 1 +end + +""" + $(SIGNATURES) +""" +function deleteBloblet!(node, label::Symbol) + !haskey(refBloblets(node), label) && throw(LabelNotFoundError("Bloblet", label)) + pop!(refBloblets(node), label) + return 1 +end + +""" + $(SIGNATURES) +List all Bloblet keys for a variable `label` in `dfg` +""" +function listBloblets(node) + return collect(keys(refBloblets(node))) +end diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index 998cca0f..54099ad3 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -31,7 +31,7 @@ const FactorCache = AbstractFactorCache # eliminated::Bool = false # TODO should eliminated and potentialused be moved outside of FactorState? potentialused::Bool = false # TODO ^ multihypo::Vector{Float64} = Float64[] # TODO re-evaluate after refactoring w #477 - certainhypo::Vector{Int} = Int[] + certainhypo::Vector{Int} = Int[] #TODO mihgt be dead code? nullhypo::Float64 = 0.0 # solveInProgress::Int = 0 #TODO maybe deprecated or move to operational memory, also why Int? inflation::Float64 = 0.0 @@ -74,13 +74,13 @@ StructUtils.@kwarg struct FactorDFG{T <: AbstractObservation, N} <: AbstractGrap variableorder::NTuple{N, Symbol} & (choosetype = x->NTuple{length(x), Symbol},) # NOTE v0.29 renamed from _variableOrderSymbols """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::NanoDate = ndnow(UTC) & (lower = timestamp,) # NOTE v0.29 changed from ZonedDateTime + timestamp::TimeDateZone = TimeDateZone(now(localzone())) # NOTE v0.29 changed from ZonedDateTime # TODO # """(Optional) Steady (monotonic) time in nanoseconds `Nanosecond` (`Int64``)""" # nstime::Nanosecond #NOTE v0.29 REMOVED as not used, add when needed, or now as steadytime. """Solvable flag for the factor. Accessors: [`getSolvable`](@ref), [`setSolvable!`](@ref)""" - solvable::Base.RefValue{Int} = Ref(1) & (lower = getindex, lift = Ref) + solvable::Base.RefValue{Int} = Ref{Int}(1) #& (lower = getindex, lift = Ref) """Dictionary of small data associated with this variable. Accessors: [`getBloblet`](@ref), [`addBloblet!`](@ref)""" bloblets::Bloblets = Bloblets() #NOTE v0.29 changed from smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}() @@ -102,16 +102,13 @@ end version(::Type{<:FactorDFG}) = v"0.29.0" -#FIXME use style to avoid type piracy -StructUtils.structlike(::JSON.JSONStyle, ::Type{Base.RefValue{Int64}}) = false - ##------------------------------------------------------------------------------ ## Constructors function FactorDFG( variableorder::Union{<:Tuple, Vector{Symbol}}, observation::AbstractObservation; label::Symbol = assembleFactorName(variableorder), - timestamp::Union{NanoDate, ZonedDateTime} = ndnow(UTC), + timestamp::Union{TimeDateZone, ZonedDateTime} = TimeDateZone(now(localzone())), tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}([:FACTOR]), bloblets::Bloblets = Bloblets(), multihypo::Vector{Float64} = Float64[], @@ -132,10 +129,10 @@ function FactorDFG( if timestamp isa ZonedDateTime Base.depwarn( - "`FactorDFG` timestamp as `ZonedDateTime` is deprecated, use `NanoDate` instead", + "`FactorDFG` timestamp as `ZonedDateTime` is deprecated, use `TimeDateZone` instead", :FactorDFG, ) - timestamp = NanoDate(timestamp.utc_datetime) + timestamp = TimeDateZone(timestamp.utc_datetime) end # create factor data @@ -165,7 +162,9 @@ function FactorDFG( state::FactorState = FactorState(), cache = nothing; tags::Set{Symbol} = Set{Symbol}([:FACTOR]), - timestamp::Union{DateTime, ZonedDateTime, NanoDate} = ndnow(UTC), + timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = TimeDateZone( + now(localzone()), + ), solvable::Int = 1, bloblets::Bloblets = Bloblets(), blobentries::Blobentries = Blobentries(), @@ -189,22 +188,16 @@ function FactorDFG( solvercache = Ref(cache) end - # deprecated in v0.29 + # TODO v0.29 deprecate ZonedDateTime or convert internally if timestamp isa ZonedDateTime - Base.depwarn( - "`FactorDFG` timestamp as `ZonedDateTime` is deprecated, use `NanoDate(timestamp.utc_datetime)` instead", - :FactorDFG, - ) - nd_timestamp = NanoDate(timestamp.utc_datetime) - else - nd_timestamp = timestamp + timestamp = TimeDateZone(timestamp) end return FactorDFG( label, tags, Tuple(variableorder), - nd_timestamp, + timestamp, Ref(solvable), bloblets, observation, @@ -239,13 +232,13 @@ Base.@kwdef struct FactorSummary <: AbstractGraphFactor variableorder::Tuple{Vararg{Symbol}} #TODO changed to NTuple """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::NanoDate + timestamp::TimeDateZone end function FactorSummary( label::Symbol, variableorder::Union{Vector{Symbol}, Tuple}; - timestamp::NanoDate = ndnow(UTC), + timestamp::TimeDateZone = TimeDateZone(now(localzone())), tags::Set{Symbol} = Set{Symbol}(), ) return FactorSummary(label, tags, Tuple(variableorder), timestamp) diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index 7fb3c80b..0e93e255 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -20,11 +20,11 @@ N: Manifold dimension. Fields: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct State{T <: StateType, P, N} +@kwdef mutable struct State{T <: StateType, P, N} """ - Globally unique identifier. + Identifier associated with this State object. """ - id::Union{UUID, Nothing} = nothing # If it's blank it doesn't exist in the DB. + label::Symbol # TODO renamed from solveKey """ Vector of on-manifold points used to represent a ManifoldKernelDensity (or parametric) belief. """ @@ -36,15 +36,14 @@ Base.@kwdef mutable struct State{T <: StateType, P, N} "Parametric (Gaussian) covariance." covar::Vector{SMatrix{N, N, Float64}} = SMatrix{getDimension(T), getDimension(T), Float64}[] - BayesNetOutVertIDs::Vector{Symbol} = Symbol[] #TODO looks unused? - dimIDs::Vector{Int} = Int[] # TODO Likely deprecate + # BayesNetOutVertIDs::Vector{Symbol} = Symbol[] #TODO looks unused? - dims::Int = getDimension(T) #TODO should we deprecate in favor of N - """ - Flag used by junction (Bayes) tree construction algorithm to know whether this variable has yet been included in the tree construction. - """ - eliminated::Bool = false - BayesNetVertID::Symbol = :NOTHING # Union{Nothing, } #TODO deprecate + # dims::Int = getDimension(T) #TODO should we deprecate in favor of N + # """ + # Flag used by junction (Bayes) tree construction algorithm to know whether this variable has yet been included in the tree construction. + # """ + # eliminated::Bool = false + # BayesNetVertID::Symbol = :NOTHING # Union{Nothing, } #TODO deprecate separator::Vector{Symbol} = Symbol[] """ False if initial numerical values are not yet available or stored values are not ready for further processing yet. @@ -53,32 +52,25 @@ Base.@kwdef mutable struct State{T <: StateType, P, N} """ Stores the amount information (per measurement dimension) captured in each coordinate dimension. """ - infoPerCoord::Vector{Float64} = zeros(getDimension(T)) - """ - Should this variable solveKey be treated as marginalized in inference computations. + observability::Vector{Float64} = Float64[]#zeros(getDimension(T)) #TODO renamed from infoPerCoord """ - ismargin::Bool = false + Should this state be treated as marginalized in inference computations. """ - Should this variable solveKey always be kept fluid and not be automatically marginalized. - """ - dontmargin::Bool = false + marginalized::Bool = false #TODO renamed from ismargin # """ - # Convenience flag on whether a solver is currently busy working on this variable solveKey. + # Should this variable solveKey always be kept fluid and not be automatically marginalized. # """ - # solveInProgress::Int = 0 - """ - How many times has a solver updated this variable solveKey estimte. - """ - solvedCount::Int = 0 - """ - solveKey identifier associated with this State object. - """ - solveKey::Symbol = :default + # dontmargin::Bool = false """ - Future proofing field for when more multithreading operations on graph nodes are implemented, these conditions are meant to be used for atomic write transactions to this VND. + How many times has a solver updated this state estimate. """ - events::Dict{Symbol, Threads.Condition} = Dict{Symbol, Threads.Condition}() + solves::Int = 0 # TODO renamed from solvedCount + # """ + # Future proofing field for when more multithreading operations on graph nodes are implemented, these conditions are meant to be used for atomic write transactions to this VND. + # """ + # events::Dict{Symbol, Threads.Condition} = Dict{Symbol, Threads.Condition}() # + statetype::Symbol = Symbol(stringVariableType(T())) end ##------------------------------------------------------------------------------ @@ -86,8 +78,8 @@ end function State{T}(; kwargs...) where {T <: StateType} return State{T, getPointType(T), getDimension(T)}(; kwargs...) end -function State(variableType::StateType; kwargs...) - return State{typeof(variableType)}(; kwargs...) +function State(label, variableType::StateType; kwargs...) + return State{typeof(variableType)}(; label, kwargs...) end function State(state::State; kwargs...) @@ -96,171 +88,51 @@ function State(state::State; kwargs...) kwargs..., ) end -##============================================================================== -## PackedState.jl -##============================================================================== - -""" -$(TYPEDEF) -Packed State structure for serializing DFGVariables. - - --- -Fields: -$(TYPEDFIELDS) -""" -Base.@kwdef mutable struct PackedState - id::Union{UUID, Nothing} # If it's blank it doesn't exist in the DB. - vecval::Vector{Float64} - dimval::Int - vecbw::Vector{Float64} - dimbw::Int - BayesNetOutVertIDs::Vector{Symbol} # Int - dimIDs::Vector{Int} - dims::Int - eliminated::Bool # TODO Questionable usage, set but never read? - BayesNetVertID::Symbol # Int #TODO deprecate - separator::Vector{Symbol} # Int #TODO maybe remove from State and have in variable only. - variableType::String - initialized::Bool - infoPerCoord::Vector{Float64} - ismargin::Bool - dontmargin::Bool - solvedCount::Int - solveKey::Symbol - covar::Vector{Float64} - _version::VersionNumber = _getDFGVersion() -end - -#FIXME remove once solveKey field is renamed to `label` -getLabel(packedstate::PackedState) = packedstate.solveKey - -# maybe add -# createdTimestamp::DateTime#! -# lastUpdatedTimestamp::DateTime#! -##============================================================================== -## PointParametricEst -##============================================================================== +StructUtils.structlike(::Type{<:State}) = false +StructUtils.lower(state::State) = DFG.packState(state) +StructUtils.lift(::Type{<:State}, obj) = DFG.unpackState(obj) ##------------------------------------------------------------------------------ -## AbstractPointParametricEst interface -##------------------------------------------------------------------------------ +## States - OrderedDict{Symbol, State} +const States = OrderedDict{Symbol, State{T, P, N}} where {T <: AbstractStateType, P, N} -abstract type AbstractPointParametricEst end - -##------------------------------------------------------------------------------ -## MeanMaxPPE -##------------------------------------------------------------------------------ -""" - $TYPEDEF +StructUtils.dictlike(::Type{<:States}) = false +StructUtils.structlike(::Type{<:States}) = false +StructUtils.arraylike(::Type{<:States}) = false -Data container to store Parameteric Point Estimate (PPE) for mean and max. -""" -Base.@kwdef struct MeanMaxPPE <: AbstractPointParametricEst - id::Union{UUID, Nothing} = nothing # If it's blank it doesn't exist in the DB. - solveKey::Symbol - suggested::Vector{Float64} - max::Vector{Float64} - mean::Vector{Float64} - _type::String = "MeanMaxPPE" - _version::VersionNumber = _getDFGVersion() - createdTimestamp::Union{ZonedDateTime, Nothing} = nothing - lastUpdatedTimestamp::Union{ZonedDateTime, Nothing} = nothing +function StructUtils.lower(states::States) + return map(collect(values(states))) do (state) + return StructUtils.lower(state) + end end -##------------------------------------------------------------------------------ -## Constructors - -function MeanMaxPPE( - solveKey::Symbol, - suggested::Vector{Float64}, - max::Vector{Float64}, - mean::Vector{Float64}, -) - return MeanMaxPPE( - nothing, - solveKey, - suggested, - max, - mean, - "MeanMaxPPE", - _getDFGVersion(), - now(tz"UTC"), - now(tz"UTC"), - ) +function StructUtils.lift( + ::StructUtils.StructStyle, + S::Type{<:States{T}}, + json_vector::Vector, +) where {T} + states = S() + foreach(json_vector) do obj + return push!(states, Symbol(obj.label) => StructUtils.make(State{T}, obj)) + end + return states, nothing end -## Metadata -""" - $SIGNATURES -Return the fields of MeanMaxPPE that are estimates. -NOTE: This is needed for each AbstractPointParametricEst. -Closest we can get to a decorator pattern. -""" -getEstimateFields(::MeanMaxPPE) = [:suggested, :max, :mean] - ##============================================================================== ## DFG Variables ##============================================================================== -""" - $(TYPEDEF) - -The Variable information packed in a way that accomdates multi-lang using json. - -Notes: -- timestamp is a `ZonedDateTime` in UTC. -- nstime can be used as mission time, with the convention that the timestamp millis coincide with the mission start nstime - - e.g. timestamp is `2020-01-01 06:30:01.250 UTC` and first nstime is `250_000_000`. -""" -Base.@kwdef struct VariableDFG <: AbstractGraphVariable - id::Union{UUID, Nothing} = nothing - label::Symbol - tags::Vector{Symbol} = Symbol[] - timestamp::ZonedDateTime = now(tz"UTC") - nstime::String = "0" - ppes::Vector{MeanMaxPPE} = MeanMaxPPE[] - blobEntries::Vector{Blobentry} = Blobentry[] - variableType::String - _version::VersionNumber = _getDFGVersion() - metadata::String = "e30=" - solvable::Int = 1 - solverData::Vector{PackedState} = PackedState[] -end -# maybe add to variable -# createdTimestamp::DateTime -# lastUpdatedTimestamp::DateTime - -#IIF like contruction helper for packed variable -function VariableDFG( - label::Symbol, - variableType::String; - tags::Vector{Symbol} = Symbol[], - timestamp::ZonedDateTime = now(tz"UTC"), - solvable::Int = 1, - nanosecondtime::Int64 = 0, - smalldata::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), - kwargs..., -) - union!(tags, [:VARIABLE]) - - pacvar = VariableDFG(; - label, - variableType, - nstime = string(nanosecondtime), - solvable, - tags, - metadata = base64encode(JSON.json(smalldata)), - timestamp, - kwargs..., - ) - - return pacvar -end - ##------------------------------------------------------------------------------ -## VariableCompute lv2 +## VariableCompute ##------------------------------------------------------------------------------ +# The Variable information packed in a way that accomdates multi-lang using json. + +# Notes: +# - timestamp is a `ZonedDateTime` in UTC. +# - nstime can be used as mission time, with the convention that the timestamp millis coincide with the mission start nstime +# - e.g. timestamp is `2020-01-01 06:30:01.250 UTC` and first nstime is `250_000_000`. + """ $(TYPEDEF) Complete variable structure for a DistributedFactorGraph variable. @@ -269,71 +141,122 @@ Complete variable structure for a DistributedFactorGraph variable. Fields: $(TYPEDFIELDS) """ -Base.@kwdef struct VariableCompute{T <: StateType, P, N} <: AbstractGraphVariable - """The ID for the variable""" - id::Union{UUID, Nothing} = nothing +@kwdef struct VariableDFG{T <: StateType, P, N} <: AbstractGraphVariable """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::ZonedDateTime = now(localzone()) + timestamp::TimeDateZone = TimeDateZone(now(localzone())) #NOTE changed to TimeDateZone in v0.29 """Nanoseconds since a user-understood epoch (i.e unix epoch, robot boot time, etc.)""" - nstime::Nanosecond = Nanosecond(0) + steadytime::Union{Nothing, Nanosecond} = nothing #NOTE changed to TimeDateZone in v0.29 + #nstime::String = "0" #NOTE different uses, as 0-999_999 nanosecond part of timestamp now in timestamp, as steady timestamp now in steadytime """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`getTags`](@ref), [`mergeTags!`](@ref), and [`removeTags!`](@ref)""" tags::Set{Symbol} = Set{Symbol}() - """Dictionary of parametric point estimates keyed by solverDataDict keys - Accessors: [`addPPE!`](@ref), [`updatePPE!`](@ref), and [`deletePPE!`](@ref)""" - ppeDict::Dict{Symbol, AbstractPointParametricEst} = - Dict{Symbol, AbstractPointParametricEst}() - """Dictionary of solver data. May be a subset of all solutions if a solver label was specified in the get call. + """Dictionary of state data. May be a subset of all solutions if a solver label was specified in the get call. Accessors: [`addState!`](@ref), [`mergeState!`](@ref), and [`deleteState!`](@ref)""" - solverDataDict::Dict{Symbol, State{T, P, N}} = Dict{Symbol, State{T, P, N}}() + states::OrderedDict{Symbol, State{T, P, N}} = OrderedDict{Symbol, State{T, P, N}}() #NOTE field renamed from solverDataDict in v0.29 """Dictionary of small data associated with this variable. Accessors: [`getBloblet`](@ref), [`setBloblet!`](@ref)""" - smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}() + bloblets::Bloblets = Bloblets() #NOTE changed from smallData in v0.29 """Dictionary of large data associated with this variable. Accessors: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`mergeBlobentry!`](@ref), and [`deleteBlobentry!`](@ref)""" - dataDict::Dict{Symbol, Blobentry} = Dict{Symbol, Blobentry}() + blobentries::Blobentries = Blobentries() #NOTE renamed from dataDict in v0.29 """Solvable flag for the variable. Accessors: [`getSolvable`](@ref), [`setSolvable!`](@ref)""" - solvable::Base.RefValue{Int} = Ref(1) + solvable::Base.RefValue{Int} = Ref{Int}(1) #& (lower = getindex,) + statetype::Symbol = Symbol(stringVariableType(T())) + # TODO autotype or version and statetype + _autotype::Nothing = nothing #& (name = :type, lower = _ -> TypeMetadata(VariableDFG)) +end +version(::Type{<:VariableDFG}) = v"0.29" +refStates(v::VariableDFG) = v.states + +#NOTE fielddefaults and fieldtags not through @kwarg macro due to error with State{T, P, N} +function StructUtils.fielddefaults( + ::StructUtils.StructStyle, + ::Type{VariableDFG{T, P, N}}, +) where {T, P, N} + return ( + timestamp = TimeDateZone(now(localzone())), + tags = Set{Symbol}(), + # states = OrderedDict{Symbol, State{T, P, N}}(), + bloblets = Bloblets(), + blobentries = Blobentries(), + solvable = Ref(1), + _autotype = nothing, + ) +end + +function StructUtils.fieldtags(::StructUtils.StructStyle, ::Type{<:VariableDFG}) + return ( + _autotype = (name = :type, lower = _ -> TypeMetadata(VariableDFG)), + # solvable = (lower = getindex,), + ) +end + +function resolveVariableType(lazyobj) + T = parseVariableType(lazyobj.statetype[]) + return VariableDFG{T, getPointType(T), getDimension(T)} end -refStates(v::VariableCompute) = v.solverDataDict -refMetadata(v::VariableCompute) = v.smallData -refBlobentries(v::VariableCompute) = v.dataDict +@choosetype VariableDFG resolveVariableType +# JSON.omit_empty(::DistributedFactorGraphs.DFGJSONStyle, ::Type{<:VariableDFG}) = true + +const VariableCompute = VariableDFG ##------------------------------------------------------------------------------ ## Constructors """ $SIGNATURES -The default VariableCompute constructor. +The default VariableDFG constructor. """ -function VariableCompute( +#IIF like contruction helper for VariableDFG +function VariableDFG( label::Symbol, - T::Type{<:StateType}; - timestamp::ZonedDateTime = now(localzone()), - solvable::Union{Int, Base.RefValue{Int}} = Ref(1), + statetype::Union{T, Type{T}}; + tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}(), + timestamp::Union{TimeDateZone, ZonedDateTime} = TimeDateZone(now(localzone())), + solvable::Union{Int, Base.RefValue{Int}} = Ref{Int}(1), + steadytime::Union{Nothing, Nanosecond} = nothing, + nanosecondtime = nothing, + smalldata = nothing, kwargs..., -) - solvable isa Int && (solvable = Ref(solvable)) +) where {T <: StateType} + if timestamp isa ZonedDateTime + # TODO @warn + timestamp = TimeDateZone(timestamp) + end + if !isnothing(nanosecondtime) + @assert isnothing(steadytime), + "nanosecondtime is replaced by steadytime. Cannot specify both steadytime and nanosecondtime" + Base.depwarn( + "nanosecondtime kwarg is deprecated, use steadytime instead", + :VariableDFG, + ) + steadytime = Nanosecond(nanosecondtime) + end + if !isnothing(smalldata) + Base.depwarn("smalldata kwarg is deprecated, use bloblets instead", :VariableDFG) + #TODO convert smalldata to bloblets + end + if solvable isa Int + solvable = Ref(solvable) + end + union!(tags, [:VARIABLE]) N = getDimension(T) P = getPointType(T) - return VariableCompute{T, P, N}(; label, timestamp, solvable, kwargs...) -end - -function VariableCompute(label::Symbol, variableType::StateType; kwargs...) - return VariableCompute(label, typeof(variableType); kwargs...) + return VariableDFG{T, P, N}(; label, steadytime, solvable, tags, timestamp, kwargs...) end -function VariableCompute(label::Symbol, solverData::State; kwargs...) - return VariableCompute(; +function VariableDFG(label::Symbol, state::State; kwargs...) + return VariableDFG( label, - solverDataDict = Dict(:default => solverData), + getStateType(state); + states = OrderedDict(state.label => state), kwargs..., ) end @@ -367,38 +290,21 @@ Fields: $(TYPEDFIELDS) """ @tags struct VariableSummary <: AbstractGraphVariable - """The ID for the variable""" - id::Union{UUID, Nothing} """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::ZonedDateTime + timestamp::TimeDateZone """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`getTags`](@ref), [`mergeTags!`](@ref), and [`removeTags!`](@ref)""" tags::Set{Symbol} - """Dictionary of parametric point estimates keyed by solverDataDict keys - Accessors: [`addPPE!`](@ref), [`updatePPE!`](@ref), and [`deletePPE!`](@ref)""" - ppeDict::Dict{Symbol, <:AbstractPointParametricEst} - """Symbol for the variableType for the underlying variable. - Accessor: [`getVariableType`](@ref)""" - variableTypeName::Symbol & (json = (name = "variableType",)) # TODO check from StructTypes.names(::Type{VariableSummary}) = ((:variableTypeName, :variableType),) + """Symbol for the state type for the underlying variable. + Accessor: [`getStateType`](@ref)""" + statetype::Symbol """Dictionary of large data associated with this variable. Accessors: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`mergeBlobentry!`](@ref), and [`deleteBlobentry!`](@ref)""" - dataDict::Dict{Symbol, Blobentry} -end - -function VariableSummary(id, label, timestamp, tags, ::Nothing, variableTypeName, ::Nothing) - return VariableSummary( - id, - label, - timestamp, - tags, - Dict{Symbol, MeanMaxPPE}(), - variableTypeName, - Dict{Symbol, Blobentry}(), - ) + blobentries::Blobentries end ##------------------------------------------------------------------------------ @@ -414,8 +320,6 @@ Fields: $(TYPEDFIELDS) """ Base.@kwdef struct VariableSkeleton <: AbstractGraphVariable - """The ID for the variable""" - id::Union{UUID, Nothing} = nothing """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol @@ -424,30 +328,24 @@ Base.@kwdef struct VariableSkeleton <: AbstractGraphVariable tags::Set{Symbol} = Set{Symbol}() end -function VariableSkeleton( - label::Symbol, - tags = Set{Symbol}(); - id::Union{UUID, Nothing} = nothing, -) - return VariableSkeleton(id, label, tags) +function VariableSkeleton(label::Symbol, tags = Set{Symbol}();) + return VariableSkeleton(label, tags) end ##============================================================================== ## Conversion constructors ##============================================================================== -function VariableSummary(v::VariableCompute) +function VariableSummary(v::VariableCompute{T}) where {T} return VariableSummary( - v.id, v.label, v.timestamp, copy(v.tags), - deepcopy(v.ppeDict), - Symbol(typeof(getVariableType(v))), - v.dataDict, + Symbol(stringVariableType(T())), + copy(v.blobentries), ) end function VariableSkeleton(v::AbstractGraphVariable) - return VariableSkeleton(v.id, v.label, copy(v.tags)) + return VariableSkeleton(v.label, copy(v.tags)) end diff --git a/src/serialization/DFGStructStyles.jl b/src/serialization/DFGStructStyles.jl new file mode 100644 index 00000000..4959ba79 --- /dev/null +++ b/src/serialization/DFGStructStyles.jl @@ -0,0 +1,13 @@ +struct DFGJSONStyle <: JSON.JSONStyle end + +StructUtils.structlike(::DFGJSONStyle, ::Type{Base.RefValue{Int}}) = false +StructUtils.lower(::DFGJSONStyle, x::Base.RefValue{Int}) = x[] +function StructUtils.lift(::DFGJSONStyle, ::Type{Base.RefValue{Int}}, x::Integer) + return Ref{Int}(x), nothing +end + +StructUtils.structlike(::DFGJSONStyle, ::Type{TimeDateZone}) = false +StructUtils.lower(::DFGJSONStyle, x::TimeDateZone) = string(x) +function StructUtils.lift(::DFGJSONStyle, ::Type{TimeDateZone}, x::AbstractString) + return TimeDateZone(x), nothing +end diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 982f285c..bb6686f0 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -441,7 +441,7 @@ function getVariable(dfg::AbstractDFG, label::Symbol, solveKey::Symbol) # function getVariable(dfg::AbstractDFG, label::Symbol; stateLabelFilter::Union{Nothing, ...} = nothing) var = getVariable(dfg, label) - if isa(var, VariableCompute) && !haskey(var.solverDataDict, solveKey) + if isa(var, VariableCompute) && !haskey(var.states, solveKey) throw(LabelNotFoundError("VariableNode", solveKey)) elseif !isa(var, VariableCompute) @warn "getVariable(dfg, label, solveKey) only supported for type VariableCompute." @@ -756,8 +756,8 @@ end Find and return the closest timestamp from two sets of Tuples. Also return the minimum delta-time (`::Millisecond`) and how many elements match from the two sets are separated by the minimum delta-time. """ function findClosestTimestamp( - setA::Vector{Tuple{ZonedDateTime, T}}, - setB::Vector{Tuple{ZonedDateTime, S}}, + setA::Vector{Tuple{TimeDateZone, T}}, + setB::Vector{Tuple{TimeDateZone, S}}, ) where {S, T} # # build matrix of delta times, ranges on rows x vars on columns @@ -798,7 +798,7 @@ ls, listVariables, findClosestTimestamp """ function findVariableNearTimestamp( dfg::AbstractDFG, - timest::ZonedDateTime, + timest::TimeDateZone,#ZonedDateTime, regexFilter::Union{Nothing, Regex} = nothing; tags::Vector{Symbol} = Symbol[], solvable::Int = 0, @@ -848,7 +848,7 @@ function findVariableNearTimestamp( ) return findVariableNearTimestamp( dfg, - ZonedDateTime(timest, timezone), + TimeDateZone(timest, timezone), regexFilter; kwargs..., ) diff --git a/src/services/CompareUtils.jl b/src/services/CompareUtils.jl index 797e9998..fc535a02 100644 --- a/src/services/CompareUtils.jl +++ b/src/services/CompareUtils.jl @@ -18,9 +18,7 @@ implement compare if needed. # Generate compares automatically for all in this union const GeneratedCompareUnion = Union{ - MeanMaxPPE, State, - PackedState, Blobentry, VariableCompute, VariableDFG, @@ -197,30 +195,30 @@ end function compare(a::State, b::State) a.val != b.val && @debug("val is not equal") === nothing && return false a.bw != b.bw && @debug("bw is not equal") === nothing && return false - a.BayesNetOutVertIDs != b.BayesNetOutVertIDs && - @debug("BayesNetOutVertIDs is not equal") === nothing && - return false - a.dimIDs != b.dimIDs && @debug("dimIDs is not equal") === nothing && return false - a.dims != b.dims && @debug("dims is not equal") === nothing && return false - a.eliminated != b.eliminated && - @debug("eliminated is not equal") === nothing && - return false - a.BayesNetVertID != b.BayesNetVertID && - @debug("BayesNetVertID is not equal") === nothing && - return false + # a.BayesNetOutVertIDs != b.BayesNetOutVertIDs && + # @debug("BayesNetOutVertIDs is not equal") === nothing && + # return false + # a.eliminated != b.eliminated && + # @debug("eliminated is not equal") === nothing && + # return false + # a.BayesNetVertID != b.BayesNetVertID && + # @debug("BayesNetVertID is not equal") === nothing && + # return false a.separator != b.separator && @debug("separator is not equal") === nothing && return false a.initialized != b.initialized && @debug("initialized is not equal") === nothing && return false - !isapprox(a.infoPerCoord, b.infoPerCoord; atol = 1e-13) && + !isapprox(a.observability, b.observability; atol = 1e-13) && @debug("infoPerCoord is not equal") === nothing && return false - a.ismargin != b.ismargin && @debug("ismargin is not equal") === nothing && return false - a.dontmargin != b.dontmargin && - @debug("dontmargin is not equal") === nothing && + a.marginalized != b.marginalized && + @debug("ismargin is not equal") === nothing && return false + # a.dontmargin != b.dontmargin && + # @debug("dontmargin is not equal") === nothing && + # return false getVariableType(a) != getVariableType(b) && @debug("variableType is not equal") === nothing && return false @@ -243,7 +241,7 @@ function compareVariable( skiplist = union( [ :attributes; - :solverDataDict; + :states; :createdTimestamp; :lastUpdatedTimestamp; :timezone; @@ -255,7 +253,7 @@ function compareVariable( varskiplist = skipsamples ? [:val; :bw] : Symbol[] skiplist = union([:variableType;], varskiplist) union!(skiplist, skip) - TP = TP && compareAll(A.solverDataDict, B.solverDataDict; skip = skiplist, show = show) + TP = TP && compareAll(A.states, B.states; skip = skiplist, show = show) Ad = getState(A, :default) #FIXME why onlly comparing default? Bd = getState(B, :default) diff --git a/src/services/CustomPrinting.jl b/src/services/CustomPrinting.jl index 720c1d53..3a1652fb 100644 --- a/src/services/CustomPrinting.jl +++ b/src/services/CustomPrinting.jl @@ -17,9 +17,9 @@ function printVariable( if short # opmemt = (getVariableType(vert) |> typeof ).name - vari = getVariableType(vert) |> typeof - printstyled(ioc, typeof(vert).name.name, "{"; bold = true) - printstyled(ioc, vari.name.name; bold = true, color = :blue) + vari = getStateType(vert) |> typeof + printstyled(ioc, nameof((typeof(vert))), "{"; bold = true) + printstyled(ioc, vari; bold = true, color = :blue) printstyled(ioc, "...}"; bold = true) println(ioc, "") # printstyled(ioc, summary(vert),"\n", bold=true) @@ -30,14 +30,13 @@ function printVariable( println(ioc, "") catch e end - vnd = if haskey(vert.solverDataDict, :default) + vnd = if hasState(vert, :default) getState(vert, :default) else nothing end - println(ioc, " ID: ", vert.id) println(ioc, " timestamp: ", vert.timestamp) - println(ioc, " nstime: ", vert.nstime) + isnothing(vert.steadytime) || println(ioc, " steadytime: ", vert.steadytime) print(ioc, " label: ") printstyled(ioc, vert.label; bold = true) println(ioc) @@ -49,7 +48,7 @@ function printVariable( # list the marginalization status ismarg = solk .|> x -> isMarginalized(vert, x) isinit = solk .|> x -> isInitialized(vert, x) - printstyled(ioc, " # VND solveKeys= ($(lsolk))"; bold = true) + printstyled(ioc, " # states: ($(lsolk))"; bold = true) println(ioc, "") printstyled(ioc, " # initialized: "; bold = true) println(ioc, "(true=", sum(isinit), ",false=", length(isinit) - sum(isinit), ")") @@ -66,24 +65,6 @@ function printVariable( printstyled(ioc, " VNDs: "; bold = true) println(ioc, solk[smsk], 4 < lsolk ? "..." : "") end - printstyled(ioc, " # PPE solveKeys= ($(length(getPPEDict(vert))))"; bold = true) - println(ioc, "") - if haskey(getPPEDict(vert), :default) - print(ioc, " :default ") - println( - ioc, - "<-- .suggested: ", - round.(getPPE(vert, :default).suggested; digits = 4), - ) - end - maxkeys = 4 - for (key, ppe) in getPPEDict(vert) - key == :default && continue # skip as default is done separately - maxkeys -= 1 - maxkeys == 0 && break - print(ioc, " :$key ") - println(ioc, "<-- .suggested: ", round.(ppe.suggested; digits = 4)) - end println(ioc, " # Blobentries: (", length(listBlobentries(vert)), ")") printstyled(ioc, " VariableType: "; color = :blue, bold = true) println(ioc, vari) @@ -93,8 +74,7 @@ function printVariable( printstyled(ioc, summary(vert); bold = true, color = :blue) println(ioc, "") - :solver in skipfields && push!(skipfields, :solverDataDict) - :ppe in skipfields && push!(skipfields, :ppeDict) + :solver in skipfields && push!(skipfields, :states) t = typeof(vert) fields = setdiff(fieldnames(t), skipfields) diff --git a/src/services/DFGVariable.jl b/src/services/DFGVariable.jl index e8c89b17..8dee71d7 100644 --- a/src/services/DFGVariable.jl +++ b/src/services/DFGVariable.jl @@ -1,68 +1,24 @@ ##============================================================================== ## Accessors ##============================================================================== -##============================================================================== -## PointParametricEst -##============================================================================== -"$(SIGNATURES)" -getPPEMax(est::AbstractPointParametricEst) = est.max -function getPPEMax(fg::AbstractDFG, varlabel::Symbol, solveKey::Symbol = :default) - return getPPE(fg, varlabel, solveKey) |> getPPEMax -end - -"$(SIGNATURES)" -getPPEMean(est::AbstractPointParametricEst) = est.mean -function getPPEMean(fg::AbstractDFG, varlabel::Symbol, solveKey::Symbol = :default) - return getPPE(fg, varlabel, solveKey) |> getPPEMean -end - -"$(SIGNATURES)" -getPPESuggested(est::AbstractPointParametricEst) = est.suggested -function getPPESuggested(var::VariableCompute, solveKey::Symbol = :default) - return getPPE(var, solveKey) |> getPPESuggested -end -function getPPESuggested(dfg::AbstractDFG, varlabel::Symbol, solveKey::Symbol = :default) - return getPPE(getVariable(dfg, varlabel), solveKey) |> getPPESuggested -end - -"$(SIGNATURES)" -getLastUpdatedTimestamp(est::AbstractPointParametricEst) = est.lastUpdatedTimestamp ##============================================================================== ## Variable Node Data ##============================================================================== -## COMMON -# getSolveInProgress -# isSolveInProgress - ##------------------------------------------------------------------------------ ## variableType ##------------------------------------------------------------------------------ """ $(SIGNATURES) -Variable nodes `variableType` information holding a variety of meta data associated with the type of variable stored in that node of the factor graph. - -Notes -- API Quirk in that this function returns and instance of `::T` not a `::Type{<:StateType}`. - -DevWork -- TODO, see IncrementalInference.jl 1228 - -Related - -getVariableType +Get the type of the variable's state, eg. `Pose2`, `Point3`, etc. as an instance of `StateType`. """ -getVariableType(::VariableCompute{T}) where {T} = T() - -getVariableType(::State{T}) where {T} = T() +getStateType(::VariableCompute{T}) where {T} = T() -# TODO: Confirm that we can switch this out, instead of retrieving the complete variable. -# getVariableType(v::VariableCompute) = getVariableType(getState(v)) +getStateType(::State{T}) where {T} = T() -# Optimized in CGDFG -getVariableType(dfg::AbstractDFG, lbl::Symbol) = getVariableType(getVariable(dfg, lbl)) +getStateType(dfg::AbstractDFG, lbl::Symbol) = getStateType(getVariable(dfg, lbl)) ##------------------------------------------------------------------------------ ## StateType @@ -160,8 +116,8 @@ end Interface function to return the `<:ManifoldsBase.AbstractManifold` object of `variableType<:StateType`. """ getManifold(::T) where {T <: StateType} = getManifold(T) -getManifold(vari::VariableCompute) = getVariableType(vari) |> getManifold -getManifold(state::State) = getVariableType(state) |> getManifold +getManifold(vari::VariableCompute) = getStateType(vari) |> getManifold +getManifold(state::State) = getStateType(state) |> getManifold # covers both <:StateType and <:AbstractObservation getManifold(dfg::AbstractDFG, lbl::Symbol) = getManifold(dfg[lbl]) @@ -256,7 +212,7 @@ Related isSolved, setSolvedCount! """ -getSolvedCount(v::State) = v.solvedCount +getSolvedCount(v::State) = v.solves function getSolvedCount(v::VariableCompute, solveKey::Symbol = :default) return getState(v, solveKey) |> getSolvedCount end @@ -273,7 +229,7 @@ Related getSolved, isSolved """ -setSolvedCount!(v::State, val::Int) = v.solvedCount = val +setSolvedCount!(v::State, val::Int) = v.solves = val function setSolvedCount!(v::VariableCompute, val::Int, solveKey::Symbol = :default) return setSolvedCount!(getState(v, solveKey), val) end @@ -295,7 +251,7 @@ Related getSolved, setSolved! """ -isSolved(v::State) = 0 < v.solvedCount +isSolved(v::State) = 0 < v.solves function isSolved(v::VariableCompute, solveKey::Symbol = :default) return getState(v, solveKey) |> isSolved end @@ -331,7 +287,7 @@ Notes: - State default `solveKey=:default` """ function isMarginalized(vert::VariableCompute, solveKey::Symbol = :default) - return getState(vert, solveKey).ismargin + return getState(vert, solveKey).marginalized end function isMarginalized(dfg::AbstractDFG, sym::Symbol, solveKey::Symbol = :default) return isMarginalized(DFG.getVariable(dfg, sym), solveKey) @@ -343,7 +299,7 @@ end Mark a variable as marginalized `true` or `false`. """ function setMarginalized!(vnd::State, val::Bool) - return vnd.ismargin = val + return vnd.marginalized = val end function setMarginalized!(vari::VariableCompute, val::Bool, solveKey::Symbol = :default) return setMarginalized!(getState(vari, solveKey), val) @@ -361,11 +317,11 @@ end ## Variables ##============================================================================== # -# | | label | tags | timestamp | ppe | variableTypeName | solvable | solverData | smallData | dataEntries | -# |---------------------|:-----:|:----:|:---------:|:---:|:----------------:|:--------:|:----------:|:---------:|:-----------:| -# | VariableSkeleton | X | X | | | | | | | | -# | VariableSummary | X | X | X | X | X | | | | X | -# | VariableCompute | X | X | x | X | | X | X | X | X | +# | | label | tags | timestamp | variableTypeName | solvable | solverData | smallData | dataEntries | +# |---------------------|:-----:|:----:|:---------:|:----------------:|:--------:|:----------:|:---------:|:-----------:| +# | VariableSkeleton | X | X | | | | | | | +# | VariableSummary | X | X | X | X | | | | X | +# | VariableCompute | X | X | x | | X | X | X | X | # ##------------------------------------------------------------------------------ @@ -402,74 +358,6 @@ end ## COMMON: -##------------------------------------------------------------------------------ -## ppeDict -##------------------------------------------------------------------------------ - -""" - $SIGNATURES - -Get the PPE dictionary for a variable. Recommended to use CRUD operations instead, [`getPPE`](@ref), [`addPPE!`](@ref), [`updatePPE!`](@ref), [`deletePPE!`](@ref). -""" -getPPEDict(v::AbstractGraphVariable) = v.ppeDict - -#TODO FIXME don't know if this should exist, should rather always update with fg object to simplify inmem vs cloud -""" - $SIGNATURES - -Get the parametric point estimate (PPE) for a variable in the factor graph. - -Notes -- Defaults on keywords `solveKey` and `method` - -Related - -getMeanPPE, getMaxPPE, getKDEMean, getKDEFit, getPPEs, getVariablePPEs -""" -function getPPE(vari::AbstractGraphVariable, solveKey::Symbol = :default) - if haskey(getPPEDict(vari), solveKey) - return getPPEDict(vari)[solveKey] - else - throw(LabelNotFoundError("PPE", solveKey, collect(keys(getPPEDict(vari))))) - end - # return haskey(ppeDict, solveKey) ? ppeDict[solveKey] : nothing -end - -""" - $SIGNATURES - -Get all the parametric point estimate (PPE) for a variable in the factor graph. -""" -function getPPEs end - -# afew more aliases on PPE, brought back from deprecated DF - -""" - $SIGNATURES - -Return full dictionary of PPEs in a variable, recommended to rather use CRUD: [`getPPE`](@ref), -""" -getVariablePPEDict(vari::AbstractGraphVariable) = getPPEDict(vari) - -""" - getVariablePPE(::VariableCompute) - getVariablePPE(::State) - -Get the Parametric Point Estimate of the given variable. -""" -getVariablePPE(args...) = getPPE(args...) - -##------------------------------------------------------------------------------ -## solverDataDict -##------------------------------------------------------------------------------ - -""" - $SIGNATURES - -Get solver data dictionary for a variable. Advised to use graph CRUD operations instead. -""" -getSolverDataDict(v::VariableCompute) = v.solverDataDict - ##------------------------------------------------------------------------------ ## Variable Metadata ##------------------------------------------------------------------------------ @@ -548,33 +436,10 @@ function emptyMetadata!(dfg::AbstractDFG, label::Symbol) end ##------------------------------------------------------------------------------ -## Data Entries and Blobs +## Blobentries and Blobs ##------------------------------------------------------------------------------ - ## see DataEntryBlob Folder -##------------------------------------------------------------------------------ -## variableTypeName -##------------------------------------------------------------------------------ -## getter in VariableSummary only -## can be utility function for others -## TODO this should return the variableType object, or try to. it should be getVariableTypeName for the accessor -## TODO Consider parameter N in variableType for dims, and storing constructor in variableTypeName -## TODO or just not having this function at all -# getVariableType(v::VariableSummary) = v.softypename() -##------------------------------------------------------------------------------ - -""" - $SIGNATURES -Retrieve the soft type name symbol for a VariableSummary. ie :Point2, Pose2, etc. -""" -getVariableTypeName(v::VariableSummary) = v.variableTypeName::Symbol - -function getVariableType(v::VariableSummary) - @warn "Looking for type in `Main`. Only use if `variableType` has only one implementation, ie. Pose2. Otherwise use the full variable." - return getfield(Main, v.variableTypeName)() -end - ##============================================================================== ## Layer 2 CRUD and SET ##============================================================================== @@ -589,16 +454,11 @@ end ##------------------------------------------------------------------------------ ## CRUD: get, add, update, delete ##------------------------------------------------------------------------------ +hasState(v::VariableCompute, label::Symbol) = haskey(v.states, label) function getState(v::VariableCompute, label::Symbol) - !haskey(getSolverDataDict(v), label) && throw(LabelNotFoundError("State", label)) - return getSolverDataDict(v)[label] -end - -function getState(v::VariableDFG, label::Symbol) - stateidx = findfirst(==(label) ∘ getLabel, v.solverData) - isnothing(stateidx) && throw(LabelNotFoundError("State", label)) - return unpackState(v.solverData[stateidx]) + !haskey(refStates(v), label) && throw(LabelNotFoundError("State", label)) + return refStates(v)[label] end """ @@ -612,7 +472,7 @@ end function getStates(dfg::AbstractDFG, variableLabel::Symbol) v = getVariable(dfg, variableLabel) - return collect(values(v.solverDataDict)) + return collect(values(v.states)) end """ @@ -625,10 +485,10 @@ function addState!(dfg::GraphsDFG, variableLabel::Symbol, state::State) end function addState!(v::VariableCompute, state::State) - if haskey(v.solverDataDict, state.solveKey) - throw(LabelExistsError("State", state.solveKey)) + if haskey(v.states, state.label) + throw(LabelExistsError("State", state.label)) end - v.solverDataDict[state.solveKey] = state + v.states[state.label] = state return state end @@ -664,10 +524,10 @@ function mergeState!(dfg::GraphsDFG, variableLabel::Symbol, vnd::State) end function mergeState!(v::VariableCompute, vnd::State) - if !haskey(v.solverDataDict, vnd.solveKey) + if !haskey(v.states, vnd.label) addState!(v, vnd) else - v.solverDataDict[vnd.solveKey] = vnd + v.states[vnd.label] = vnd end return 1 end @@ -695,16 +555,10 @@ function copytoState!( stateLabel::Symbol, state::State, ) - newstate = State( - getVariableType(state); - (k => deepcopy(getproperty(state, k)) for k in fieldnames(State))..., - solveKey = stateLabel, - ) + newstate = State(state; label = stateLabel) return mergeState!(dfg, variableLabel, newstate) end -# - """ $(SIGNATURES) Delete the variable `State` by label, returns the number of deleted elements. @@ -714,10 +568,10 @@ function deleteState!(dfg::GraphsDFG, variableLabel::Symbol, label::Symbol) end function deleteState!(v::VariableCompute, label::Symbol) - if !haskey(v.solverDataDict, label) + if !haskey(v.states, label) throw(LabelNotFoundError("State", label)) end - delete!(v.solverDataDict, label) + delete!(v.states, label) return 1 end @@ -755,7 +609,7 @@ end List all the variable state labels. """ function listStates(v::VariableCompute; labelFilter::Union{Nothing, Function} = nothing) - labels = collect(keys(v.solverDataDict)) + labels = collect(keys(v.states)) return filterDFG!(labels, labelFilter) end @@ -788,192 +642,3 @@ function listStates( end return labels end - -#TODO deprecate PPEs -##============================================================================== -## Point Parametric Estimates -##============================================================================== - -##------------------------------------------------------------------------------ -## CRUD: get, add, update, delete -##------------------------------------------------------------------------------ - -""" - $(SIGNATURES) -Get the parametric point estimate (PPE) for a variable in the factor graph for a given solve key. - -Notes -- Defaults on keywords `solveKey` and `method` - -Related -[`getPPEMean`](@ref), [`getPPEMax`](@ref), [`updatePPE!`](@ref), `mean(BeliefType)` -""" -function getPPE(v::VariableCompute, ppekey::Symbol = :default) - !haskey(v.ppeDict, ppekey) && throw(LabelNotFoundError("PPE", ppekey)) - return v.ppeDict[ppekey] -end -function getPPE(dfg::AbstractDFG, variableLabel::Symbol, ppekey::Symbol = :default) - return getPPE(getVariable(dfg, variableLabel), ppekey) -end -# Not the most efficient call but it at least reuses above (in memory it's probably ok) -function getPPE( - dfg::AbstractDFG, - sourceVariable::AbstractGraphVariable, - ppekey::Symbol = :default, -) - return getPPE(dfg, sourceVariable.label, ppekey) -end - -""" - $(SIGNATURES) -Add variable PPE, errors if it already exists. -""" -function addPPE!( - dfg::AbstractDFG, - variableLabel::Symbol, - ppe::P, -) where {P <: AbstractPointParametricEst} - var = getVariable(dfg, variableLabel) - if haskey(var.ppeDict, ppe.solveKey) - throw(LabelExistsError("PPE", ppe.solveKey)) - end - var.ppeDict[ppe.solveKey] = ppe - return ppe -end - -""" - $(SIGNATURES) -Add a new PPE entry from a deepcopy of the source variable PPE. -NOTE: Copies the PPE. -""" -function addPPE!( - dfg::AbstractDFG, - sourceVariable::VariableCompute, - ppekey::Symbol = :default, -) - return addPPE!(dfg, sourceVariable.label, deepcopy(getPPE(sourceVariable, ppekey))) -end - -function addPPEs!( - dfg::AbstractDFG, - sourceVariables::Vector{VariableCompute}, - ppekey::Symbol = :default, -) - return addPPE!.(dfg, sourceVariables, ppekey) -end - -""" - $(SIGNATURES) -Update PPE data if it exists, otherwise add it -- one call per `key::Symbol=:default`. - -Notes -- uses `ppe.solveKey` as solveKey. -""" -function updatePPE!( - dfg::AbstractDFG, - variableLabel::Symbol, - ppe::AbstractPointParametricEst; - warn_if_absent::Bool = true, -) - var = getVariable(dfg, variableLabel) - if warn_if_absent && !haskey(var.ppeDict, ppe.solveKey) - @warn "PPE '$(ppe.solveKey)' does not exist, adding" - end - #for InMemoryDFGTypes, cloud would update here - var.ppeDict[ppe.solveKey] = ppe - return ppe -end - -""" - $(SIGNATURES) -Update PPE data if it exists, otherwise add it. -NOTE: Copies the PPE data. -""" -function updatePPE!( - dfg::AbstractDFG, - sourceVariable::AbstractGraphVariable, - ppekey::Symbol = :default; - warn_if_absent::Bool = true, -) - return updatePPE!( - dfg, - sourceVariable.label, - deepcopy(getPPE(sourceVariable, ppekey)); - warn_if_absent = warn_if_absent, - ) -end - -""" - $(SIGNATURES) -Update PPE data if it exists, otherwise add it. -""" -function updatePPE!( - dfg::AbstractDFG, - sourceVariables::Vector{<:AbstractGraphVariable}, - ppekey::Symbol = :default; - warn_if_absent::Bool = true, -) - #I think cloud would do this in bulk for speed - for var in sourceVariables - updatePPE!( - dfg, - var.label, - getPPE(dfg, var, ppekey); - warn_if_absent = warn_if_absent, - ) - end -end - -""" - $(SIGNATURES) -Delete PPE data, returns the deleted element. -""" -function deletePPE!(dfg::AbstractDFG, variableLabel::Symbol, ppekey::Symbol = :default) - var = getVariable(dfg, variableLabel) - - if !haskey(var.ppeDict, ppekey) - throw(LabelNotFoundError("PPE", ppekey)) - end - pop!(var.ppeDict, ppekey) - return 1 -end - -""" - $(SIGNATURES) -Delete PPE data, returns the deleted element. -""" -function deletePPE!( - dfg::AbstractDFG, - sourceVariable::VariableCompute, - ppekey::Symbol = :default, -) - return deletePPE!(dfg, sourceVariable.label, ppekey) -end - -##------------------------------------------------------------------------------ -## SET: list, merge -##------------------------------------------------------------------------------ - -""" - $(SIGNATURES) -List all the PPE data keys in the variable. -""" -function listPPEs(dfg::AbstractDFG, variableLabel::Symbol) - v = getVariable(dfg, variableLabel) - return collect(keys(v.ppeDict))::Vector{Symbol} -end - -#TODO API and only correct level -""" - $(SIGNATURES) -Merges and updates solver and estimate data for a variable (variable can be from another graph). -Note: Makes a copy of the estimates and solver data so that there is no coupling between graphs. -""" -function mergePPEs!( - destVariable::AbstractGraphVariable, - sourceVariable::AbstractGraphVariable, -) - # We don't know which graph this came from, must be copied! - merge!(destVariable.ppeDict, deepcopy(sourceVariable.ppeDict)) - return destVariable -end diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index 640bc302..0348e968 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -42,7 +42,6 @@ function parseVariableType(_typeString::AbstractString) if isnothing(subtype) throw(SerializationError("Unable to deserialize type $(_typeString), not found")) - return nothing end if isnothing(param) @@ -57,10 +56,31 @@ end ##============================================================================== ## State Packing and unpacking ##============================================================================== - -# returns a PackedState +# Old PackedState struct fields +# id::Union{UUID, Nothing} +# vecval::Vector{Float64} +# dimval::Int +# vecbw::Vector{Float64} +# dimbw::Int +# BayesNetOutVertIDs::Vector{Symbol} +# dimIDs::Vector{Int} +# dims::Int +# eliminated::Bool +# BayesNetVertID::Symbol +# separator::Vector{Symbol} +# variableType::String +# initialized::Bool +# infoPerCoord::Vector{Float64} +# ismargin::Bool +# dontmargin::Bool +# solvedCount::Int +# solveKey::Symbol +# covar::Vector{Float64} +# _version::VersionNumber = _getDFGVersion() + +# returns a named tuple until State serialization is fully consolidated function packState(d::State{T}) where {T <: StateType} - @debug "Dispatching conversion variable -> packed variable for type $(string(getVariableType(d)))" + @debug "Dispatching conversion variable -> packed variable for type $(string(getStateType(d)))" castval = if 0 < length(d.val) precast = getCoordinates.(T, d.val) @cast castval[i, j] := precast[j][i] @@ -74,40 +94,27 @@ function packState(d::State{T}) where {T <: StateType} "Packing of more than one parametric covariance is NOT supported yet, only packing first." ) - return PackedState( - d.id, - _val, - size(castval, 1), - d.bw[:], - size(d.bw, 1), - d.BayesNetOutVertIDs, - d.dimIDs, - d.dims, - d.eliminated, - d.BayesNetVertID, - d.separator, - stringVariableType(getVariableType(d)), - d.initialized, - d.infoPerCoord, - d.ismargin, - d.dontmargin, - # d.solveInProgress, - d.solvedCount, - d.solveKey, - isempty(d.covar) ? Float64[] : vec(d.covar[1]), - _getDFGVersion(), + return ( + label = d.label, + vecval = _val, + dimval = size(castval, 1), + vecbw = d.bw[:], + dimbw = size(d.bw, 1), + separator = d.separator, + statetype = stringVariableType(getStateType(d)), + initialized = d.initialized, + observability = d.observability, + marginalized = d.marginalized, + solves = d.solves, + covar = isempty(d.covar) ? Float64[] : vec(d.covar[1]), + version = version(State), ) end -function unpackState(d::PackedState) +function unpackOldState(d) @debug "Dispatching conversion packed variable -> variable for type $(string(d.variableType))" # Figuring out the variableType - # TODO deprecated remove in v0.11 - for backward compatibility for saved variableTypes. - ststring = string(split(d.variableType, "(")[1]) - T = parseVariableType(ststring) - isnothing(T) && error( - "The variable doesn't seem to have a variableType. It needs to set up with an StateType from IIF. This will happen if you use DFG to add serialized variables directly and try use them. Please use IncrementalInference.addVariable().", - ) + T = parseVariableType(d.variableType) r3 = d.dimval c3 = r3 > 0 ? floor(Int, length(d.vecval) / r3) : 0 @@ -126,102 +133,48 @@ function unpackState(d::PackedState) # N = getDimension(T) return State{T, getPointType(T), N}(; - id = d.id, + label = Symbol(d.label), val = vals, bw = BW, #TODO only one covar is currently supported in packed VND covar = isempty(d.covar) ? SMatrix{N, N, Float64}[] : [d.covar], - BayesNetOutVertIDs = Symbol.(d.BayesNetOutVertIDs), - dimIDs = d.dimIDs, - dims = d.dims, - eliminated = d.eliminated, - BayesNetVertID = Symbol(d.BayesNetVertID), separator = Symbol.(d.separator), initialized = d.initialized, - infoPerCoord = d.infoPerCoord, - ismargin = d.ismargin, - dontmargin = d.dontmargin, - # solveInProgress = d.solveInProgress, - solvedCount = d.solvedCount, - solveKey = Symbol(d.solveKey), - events = Dict{Symbol, Threads.Condition}(), - ) -end - -##============================================================================== -## Variable Packing and unpacking -##============================================================================== - -function packVariable( - v::VariableCompute; - includePPEs::Bool = true, - includeSolveData::Bool = true, - includeDataEntries::Bool = true, -) - return VariableDFG(; - id = v.id, - label = v.label, - timestamp = v.timestamp, - nstime = string(v.nstime.value), - tags = collect(v.tags), # Symbol.() - ppes = collect(values(v.ppeDict)), - solverData = packState.(collect(values(v.solverDataDict))), - metadata = base64encode(JSON.json(v.smallData)), - solvable = getSolvable(v), - variableType = stringVariableType(DFG.getVariableType(v)), - blobEntries = collect(values(v.dataDict)), - _version = _getDFGVersion(), + observability = d.infoPerCoord, + marginalized = d.ismargin, + solves = d.solvedCount, ) end -function packVariable( - v::VariableDFG; - includePPEs::Bool = true, - includeSolveData::Bool = true, - includeDataEntries::Bool = true, -) - return v -end - -function unpackVariable(variable::VariableDFG; skipVersionCheck::Bool = false) - !skipVersionCheck && _versionCheck(variable) - - # Variable and point type - variableType = parseVariableType(variable.variableType) - isnothing(variableType) && error( - "Cannot deserialize variableType '$(variable.variableType)' in variable '$(variable.label)'", - ) - pointType = DFG.getPointType(variableType) +function unpackState(d) + @debug "Dispatching conversion packed variable -> variable for type $(string(d.statetype))" + T = parseVariableType(d.statetype) + r3 = d.dimval + c3 = r3 > 0 ? floor(Int, length(d.vecval) / r3) : 0 + M3 = reshape(d.vecval, r3, c3) + @cast val_[j][i] := M3[i, j] + vals = Vector{getPointType(T)}(undef, length(val_)) + # vals = getPoint.(T, val_) + for (i, v) in enumerate(val_) + vals[i] = getPoint(T, v) + end - ppeDict = - Dict{Symbol, MeanMaxPPE}(map(p -> p.solveKey, variable.ppes) .=> variable.ppes) + r4 = d.dimbw + c4 = r4 > 0 ? floor(Int, length(d.vecbw) / r4) : 0 + BW = reshape(d.vecbw, r4, c4) - N = getDimension(variableType) - solverDict = Dict{Symbol, State{variableType, pointType, N}}( - map(sd -> sd.solveKey, variable.solverData) .=> - map(sd -> DFG.unpackState(sd), variable.solverData), - ) - dataDict = Dict{Symbol, Blobentry}( - map(de -> de.label, variable.blobEntries) .=> variable.blobEntries, - ) - metadata = JSON.parse(base64decode(variable.metadata), Dict{Symbol, DFG.MetadataTypes}) - - return VariableCompute( - variable.label, - variableType; - id = variable.id, - timestamp = variable.timestamp, - nstime = Nanosecond(variable.nstime), - tags = Set(variable.tags), - ppeDict = ppeDict, - solverDataDict = solverDict, - smallData = metadata, - dataDict = dataDict, - solvable = variable.solvable, + # + N = getDimension(T) + return State{T, getPointType(T), N}(; + label = Symbol(d.label), + val = vals, + bw = BW, + #TODO only one covar is currently supported in packed VND + covar = isempty(d.covar) ? SMatrix{N, N, Float64}[] : [d.covar], + separator = Symbol.(d.separator), + initialized = d.initialized, + observability = d.observability, + marginalized = d.marginalized, + solves = d.solves, ) end - -VariableCompute(v::VariableCompute) = v -VariableCompute(v::VariableDFG) = unpackVariable(v) -VariableDFG(v::VariableDFG) = v -VariableDFG(v::VariableCompute) = packVariable(v) diff --git a/test/GraphsDFGSummaryTypes.jl b/test/GraphsDFGSummaryTypes.jl index b13a458a..d8b9cc46 100644 --- a/test/GraphsDFGSummaryTypes.jl +++ b/test/GraphsDFGSummaryTypes.jl @@ -5,11 +5,9 @@ dfg = GraphsDFG{NoSolverParams, VARTYPE, FACTYPE}() function DistributedFactorGraphs.VariableSummary(label::Symbol) return VariableSummary( - nothing, label, - DistributedFactorGraphs.now(localzone()), + TimeDateZone("2025-11-13T15:21:57.474125421+01:00"), Set{Symbol}(), - Dict{Symbol, MeanMaxPPE}(), :Pose2, Dict{Symbol, Blobentry}(), ) @@ -17,11 +15,9 @@ end function DistributedFactorGraphs.VariableSummary(label::Symbol, ::State{T}) where {T} return VariableSummary( - nothing, label, - DistributedFactorGraphs.now(localzone()), + TimeDateZone("2025-11-13T15:21:57.474125421+01:00"), Set{Symbol}(), - Dict{Symbol, MeanMaxPPE}(), Symbol(T), Dict{Symbol, Blobentry}(), ) @@ -32,7 +28,7 @@ function DistributedFactorGraphs.VariableSkeleton(label::Symbol, args...) end function DistributedFactorGraphs.VariableSkeleton(label::Symbol, ::State{T}) where {T} - return VariableSkeleton(nothing, label, Set{Symbol}()) + return VariableSkeleton(label, Set{Symbol}()) end dfg = GraphsDFG{NoSolverParams, VARTYPE, FACTYPE}() @@ -107,14 +103,9 @@ end if VARTYPE == VariableSummary @test getTimestamp(v1) == v1.timestamp - @test getVariableTypeName(v1) == :Pose2 end end -@testset "Updating Nodes" begin - VARTYPE == VariableSummary && PPETestBlock!(dfg, v1) -end - @testset "Adjacency Matrices" begin fg = GraphsDFG{NoSolverParams, VARTYPE, FACTYPE}() addVariable!(fg, VARTYPE(:a)) diff --git a/test/compareTests.jl b/test/compareTests.jl index 4d254305..fe3419be 100644 --- a/test/compareTests.jl +++ b/test/compareTests.jl @@ -8,9 +8,9 @@ using Dates ## Generated compare functions # State -vnd1 = State(TestVariableType1()) +vnd1 = State(:default, TestVariableType1()) vnd2 = deepcopy(vnd1) -vnd3 = State(TestVariableType2()) +vnd3 = State(:default, TestVariableType2()) @test vnd1 == vnd2 push!(vnd1.val, [1.0;]) @@ -20,16 +20,6 @@ vnd2.val[1] = [0.1;] @test !(vnd1 == vnd2) @test !(vnd1 == vnd3) -# MeanMaxPPE -ppe1 = MeanMaxPPE(:default, [1.0], [2.0], [3.0]) -ppe2 = deepcopy(ppe1) -ppe3 = MeanMaxPPE(:default, [2.0], [3.0], [4.0]) - -@test ppe1 == ppe2 -ppe2.max[1] = 0.1 -@test !(ppe1 == ppe2) -@test !(ppe1 == ppe3) - # VariableCompute v1 = VariableCompute(:x1, TestVariableType1()) v2 = deepcopy(v1) @@ -59,9 +49,9 @@ f3 = FactorCompute(:f1, [:b, :a], TestFunctorInferenceType1()) @test !(f1 == f3) ## Compare functions -vnd1 = State(TestVariableType1()) +vnd1 = State(:default, TestVariableType1()) vnd2 = deepcopy(vnd1) -vnd3 = State(TestVariableType2()) +vnd3 = State(:default, TestVariableType2()) @test compare(vnd1, vnd2) @test !compare(vnd1, vnd3) diff --git a/test/fileDFGTests.jl b/test/fileDFGTests.jl index 3c181d2d..f7735ab6 100644 --- a/test/fileDFGTests.jl +++ b/test/fileDFGTests.jl @@ -30,7 +30,7 @@ using UUIDs label = :testing, blobstore = :store, # timestamp = now(localzone()), - timestamp = DFG.ndnow(UTC), + timestamp = DFG.TimeDateZone(now(localzone())), ), ), verts, @@ -43,24 +43,12 @@ using UUIDs label = :testing2, blobstore = :store, # timestamp = ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"), - timestamp = DFG.NanoDate(2014, 5, 30, 21), + timestamp = DFG.TimeDateZone(ZonedDateTime(2014, 5, 30, 21, tz"UTC-4")), ), ), verts, ) - # Add some PPEs - ppe1 = MeanMaxPPE(:default, [1.0], [0.2], [3.456]) - ppe2 = MeanMaxPPE(; - solveKey = :other, - suggested = [1.0], - mean = [0.2], - max = [3.456], - createdTimestamp = ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"), - ) - map(v -> addPPE!(dfg, getLabel(v), deepcopy(ppe1)), verts) - map(v -> addPPE!(dfg, getLabel(v), deepcopy(ppe2)), verts) - #call update to set it on cloud mergeVariable!.(dfg, verts) @@ -92,8 +80,8 @@ using UUIDs blobid = uuid4(), label = :testing2, blobstore = :store, - # timestamp = NanoDate(2023, 2, 3, 20, tz"UTC+1"), - timestamp = DFG.NanoDate(2023, 2, 3, 20), + # timestamp = ZonedDateTime(2023, 2, 3, 20, tz"UTC+1"), + timestamp = DFG.TimeDateZone(ZonedDateTime(2023, 2, 3, 20, tz"UTC+1")), ) addGraphBlobentry!(dfg, be) @@ -136,10 +124,6 @@ using UUIDs for v in ls(dfg) @test getBlobentries(getVariable(dfg, v)) == getBlobentries(getVariable(retDFG, v)) - @test issetequal(listPPEs(dfg, v), listPPEs(retDFG, v)) - for ppe in listPPEs(dfg, v) - @test getPPE(dfg, v, ppe) == getPPE(retDFG, v, ppe) - end end # test the duplicate order #581 diff --git a/test/iifCompareTests.jl b/test/iifCompareTests.jl index 9daf5e1c..0cdb5cff 100644 --- a/test/iifCompareTests.jl +++ b/test/iifCompareTests.jl @@ -55,7 +55,7 @@ using Test fg, fg2, skipsamples = true, - skip = Symbol[:infoPerCoord; :initialized; :inferdim; :ppeDict; :solvedCount], + skip = Symbol[:infoPerCoord; :initialized; :inferdim; :solvedCount], ) # fg2 has been solved, so it should fail on the estimate dictionary @test !compareSimilarVariables( @@ -87,7 +87,6 @@ using Test :infoPerCoord :initialized :inferdim - :ppeDict :solvedCount ], ) diff --git a/test/iifInterfaceTests.jl b/test/iifInterfaceTests.jl index 79a2baec..9cd586c3 100644 --- a/test/iifInterfaceTests.jl +++ b/test/iifInterfaceTests.jl @@ -177,12 +177,8 @@ end @test getLabel(v1) == v1.label @test getTags(v1) == v1.tags @test getTimestamp(v1) == v1.timestamp - @test getVariablePPEDict(v1) == v1.ppeDict - @test_throws LabelNotFoundError DistributedFactorGraphs.getVariablePPE(v1, :notfound) - @test getState(v1, :default) === v1.solverDataDict[:default] - @test getSolverDataDict(v1) == v1.solverDataDict - # legacy compat test - @test getVariablePPEDict(v1) == v1.ppeDict # changed to .ppeDict -- delete by DFG v0.7 + @test getState(v1, :default) === v1.states[:default] + @test refStates(v1) == v1.states @test typeof(getVariableType(v1)) == Position{1} @test typeof(getVariableType(v2)) == Position{1} @@ -408,14 +404,8 @@ end # Check all fields are equal for all variables for v in ls(summaryGraph) for field in variableFields - if field != :variableTypeName - @test getproperty(getVariable(dfg, v), field) == - getfield(getVariable(summaryGraph, v), field) - else - # Special case to check the symbol variableType is equal to the full variableType. - @test Symbol(typeof(getVariableType(getVariable(dfg, v)))) == - getVariableTypeName(getVariable(summaryGraph, v)) - end + @test getproperty(getVariable(dfg, v), field) == + getfield(getVariable(summaryGraph, v), field) end end for f in lsf(summaryGraph) diff --git a/test/interfaceTests.jl b/test/interfaceTests.jl index d41fafec..5a7426b8 100644 --- a/test/interfaceTests.jl +++ b/test/interfaceTests.jl @@ -6,7 +6,7 @@ if false using Dates using UUIDs using TimeZones - using NanoDates + using TimesDates include("testBlocks.jl") @@ -22,6 +22,9 @@ if false global_logger(logger) end +# DFG.@usingDFG true +# include("testBlocks.jl") + # DFG Accessors @testset "DFG Structure and Accessors" begin # Constructors @@ -64,13 +67,13 @@ end @test printVariable(iobuf, var1; skipfields = [:timestamp, :solver, :ppe, :nstime]) === nothing - @test String(take!(iobuf)) == - "VariableCompute{TestVariableType1, Vector{Float64}, 1}\nid:\nnothing\nlabel:\n:a\ntags:\nSet([:VARIABLE, :POSE])\nsmallData:\nDict{Symbol, Union{Bool, Float64, Int64, Vector{Bool}, Vector{Float64}, Vector{Int64}, Vector{String}, String}}()\ndataDict:\nDict{Symbol, Blobentry}()\nsolvable:\nRefValue{Int64}(0)\n" - # "VariableCompute{TestVariableType1, Vector{Float64}, 1}\nid:\nnothing\nlabel:\n:a\ntags:\nSet([:VARIABLE, :POSE])\nsmallData:\nDict{Symbol, Union{Bool, Float64, Int64, Vector{Bool}, Vector{Float64}, Vector{Int64}, Vector{String}, String}}(:small=>\"data\")\ndataDict:\nDict{Symbol, Blobentry}()\nsolvable:\n0\n" + varstr = String(take!(iobuf)) + @test occursin(r"VariableDFG", varstr) + @test occursin(r"label", varstr) @test printVariable(iobuf, var1; short = true) === nothing varstr = String(take!(iobuf)) - @test occursin(r"VariableCompute", varstr) + @test occursin(r"VariableDFG", varstr) @test occursin(r"timestamp", varstr) @test occursin(r"label", varstr) @test occursin(r"bandwidths", varstr) @@ -111,10 +114,6 @@ end tagsTestBlock!(fg1, var1, v1_tags) end -@testset "Parametric Point Estimates" begin - PPETestBlock!(fg1, var1) -end - @testset "Variable Solver Data" begin VSDTestBlock!(fg1, var1) end diff --git a/test/runtests.jl b/test/runtests.jl index 6a7adf04..d5427648 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ using DistributedFactorGraphs using Pkg using Dates using TimeZones +using TimesDates using SHA using UUIDs using Aqua @@ -182,13 +183,6 @@ DFG.@usingDFG true end @testset "Testing Code Quality with Aqua" begin - Aqua.test_ambiguities([DistributedFactorGraphs]) - Aqua.test_unbound_args(DistributedFactorGraphs) - Aqua.test_undefined_exports(DistributedFactorGraphs) - Aqua.test_piracies(DistributedFactorGraphs) - Aqua.test_project_extras(DistributedFactorGraphs) - Aqua.test_stale_deps(DistributedFactorGraphs; ignore = [:Colors]) - Aqua.test_deps_compat(DistributedFactorGraphs) - # Aqua.test_project_toml_formatting(DistributedFactorGraphs) # deprecated in Aqua.jl v0.8 + Aqua.test_all(DistributedFactorGraphs) end end diff --git a/test/testBlocks.jl b/test/testBlocks.jl index 7c7aad0f..f5775f80 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -35,15 +35,15 @@ TestFunctorInferenceType1() = TestFunctorInferenceType1(TestBelief()) TestFunctorInferenceType2() = TestFunctorInferenceType2(TestBelief()) TestAbstractPrior() = TestAbstractPrior(TestBelief()) -struct PackedNothingDistribution <: AbstractPackedBelief - _type::Symbol - function PackedNothingDistribution(; _type::String = "PackedNothingDistribution") - return new(Symbol(_type)) - end -end +# struct PackedNothingDistribution <: AbstractPackedBelief +# _type::Symbol +# function PackedNothingDistribution(; _type::String = "PackedNothingDistribution") +# return new(Symbol(_type)) +# end +# end -DFG.packDistribution(::Nothing) = PackedNothingDistribution() -DFG.unpackDistribution(::PackedNothingDistribution) = nothing +# DFG.packDistribution(::Nothing) = PackedNothingDistribution() +# DFG.unpackDistribution(::PackedNothingDistribution) = nothing struct TestCCW{T <: AbstractObservation} <: FactorCache usrfnc!::T @@ -88,7 +88,6 @@ function DFGStructureAndAccessors( :AGENT, :VARIABLE, :FACTOR, - :PPE, :BLOB_ENTRY, :FACTORGRAPH, ] @@ -258,12 +257,16 @@ function DFGVariableSCA() TestVariableType1(); tags = v1_tags, solvable = 0, - solverDataDict = Dict(:default => State{TestVariableType1}()), + states = Dict(:default => State{TestVariableType1}(; label = :default)), + ) + v2 = VariableCompute( + :b, + State{TestVariableType2}(; label = :default); + tags = Set([:VARIABLE, :LANDMARK]), ) - v2 = VariableCompute(:b, State{TestVariableType2}(); tags = Set([:VARIABLE, :LANDMARK])) v3 = VariableCompute( :c, - State{TestVariableType2}(); + State{TestVariableType2}(; label = :default); timestamp = ZonedDateTime("2020-08-11T00:12:03.000-05:00"), ) @@ -272,15 +275,15 @@ function DFGVariableSCA() TestVariableType1(); tags = v1_tags, solvable = 0, - solverDataDict = Dict(:default => State{TestVariableType1}()), + states = Dict(:default => State{TestVariableType1}(; label = :default)), ) - # v1.solverDataDict[:default].val[1] = [0.0;] - # v1.solverDataDict[:default].bw[1] = [1.0;] - # v2.solverDataDict[:default].val[1] = [0.0;0.0] - # v2.solverDataDict[:default].bw[1] = [1.0;1.0] - # v3.solverDataDict[:default].val[1] = [0.0;0.0] - # v3.solverDataDict[:default].bw[1] = [1.0;1.0] + # v1.states[:default].val[1] = [0.0;] + # v1.states[:default].bw[1] = [1.0;] + # v2.states[:default].val[1] = [0.0;0.0] + # v2.states[:default].bw[1] = [1.0;1.0] + # v3.states[:default].val[1] = [0.0;0.0] + # v3.states[:default].bw[1] = [1.0;1.0] @test getLabel(v1) == v1_lbl @test getTags(v1) == v1_tags @@ -291,9 +294,7 @@ function DFGVariableSCA() @test getSolvable(v2) == 1 # TODO direct use is not recommended, use accessors, maybe not export or deprecate - @test getSolverDataDict(v1) == v1.solverDataDict - - @test getPPEDict(v1) == v1.ppeDict + @test DFG.refStates(v1) == v1.states # @test getMetadata(v1) == Dict{Symbol, MetadataTypes}() @@ -320,10 +321,7 @@ function DFGVariableSCA() @test getManifold(testvar) == TranslationGroup(1) # #TODO sort out - # getPPEs # getState - # getVariablePPEs - # getVariablePPE # getSolvedCount # isSolved # setSolvedCount @@ -565,132 +563,6 @@ function tagsTestBlock!(fg, v1, v1_tags) @test !hasTagsNeighbors(fg, :abf1, [:LANDMARK, :TAG]) end -function PPETestBlock!(fg, v1) - # "Parametric Point Estimates" - - # - `getPPEs` - # **Set** - # > - `emptyPPE!` - # > - `mergePPE!` - - # Add a new PPE of type MeanMaxPPE to :x0 - ppe = MeanMaxPPE(:default, [0.0], [0.0], [0.0]) - - @test getPPEMax(ppe) === ppe.max - @test getPPEMean(ppe) === ppe.mean - @test getPPESuggested(ppe) === ppe.suggested - @test getLastUpdatedTimestamp(ppe) === ppe.lastUpdatedTimestamp - - @test addPPE!(fg, :a, ppe) == ppe - @test_throws LabelExistsError addPPE!(fg, :a, ppe) - - @test listPPEs(fg, :a) == [:default] - - # Get the data back - note that this is a reference to above. - @test getPPE(getVariable(fg, :a), :default) == ppe - @test getPPE(fg, :a, :default) == ppe - @test getPPEMean(fg, :a, :default) == ppe.mean - @test getPPEMax(fg, :a, :default) == ppe.max - @test getPPESuggested(fg, :a, :default) == ppe.suggested - - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - - @test_throws LabelNotFoundError getPPE(fg, :a, :default) - # Update add it - @test @test_logs (:warn, Regex("'$(ppe.solveKey)' does not exist")) match_mode = :any updatePPE!( - fg, - :a, - ppe, - ) == ppe - # Update update it - @test updatePPE!(fg, :a, ppe) == ppe - @test deletePPE!(fg, :a, :default) == 1 - - # manually add ppe to v1 for tests - v1.ppeDict[:default] = deepcopy(ppe) - # Bulk copy PPE's for :x1 - @test updatePPE!(fg, [v1], :default) == nothing - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - - # New interface - @test addPPE!(fg, :a, ppe) == ppe - # Update update it - @test updatePPE!(fg, :a, ppe) == ppe - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - - #FIXME copied from lower - # @test @test_deprecated getVariablePPEs(v1) == v1.ppeDict - @test_throws LabelNotFoundError getPPE(v1, :notfound) - #TODO - # @test_deprecated getVariablePPE(v1) - - # Add a new PPE of type MeanMaxPPE to :x0 - ppe = MeanMaxPPE(:default, [0.0], [0.0], [0.0]) - addPPE!(fg, :a, ppe) - @test listPPEs(fg, :a) == [:default] - # Get the data back - note that this is a reference to above. - @test getPPE(fg, :a, :default) == ppe - - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - # Update add it - updatePPE!(fg, :a, ppe) #, :default) - # Update update it - updatePPE!(fg, :a, ppe) #, :default) - - v1.ppeDict[:default] = deepcopy(ppe) - # Bulk copy PPE's for x0 and x1 - updatePPE!(fg, [v1], :default) - # Delete it - @test deletePPE!(fg, :a, :default) == 1 - - #TODO DEPRECATE - # getEstimates - # estimates - # getVariablePPEs - # getVariablePPE - - # newvar = deepcopy(v1) - # getPPEDict(newvar)[:default] = MeanMaxPPE(:default, [150.0], [100.0], [50.0]) - # @test !(getPPEDict(newvar) == getPPEDict(v1)) - # delete!(getVariablePPEs(newvar), :default) - # getVariablePPEs(newvar)[:second] = MeanMaxPPE(:second, [15.0], [10.0], [5.0]) - # @test symdiff(collect(keys(getVariablePPEs(v1))), [:default, :second]) == Symbol[] - # @test symdiff(collect(keys(getVariablePPEs(newvar))), [:second]) == Symbol[] - # # Get the source too. - # @test symdiff(collect(keys(getVariablePPEs(getVariable(dfg, :a)))), [:default, :second]) == Symbol[] - #update - - ## TODO make sure these are covered - # global dfg - # #get the variable - # var1 = getVariable(dfg, :a) - # #make a copy and simulate external changes - # newvar = deepcopy(var1) - # getVariablePPEs(newvar)[:default] = MeanMaxPPE(:default, [150.0], [100.0], [50.0]) - # #update - # mergeUpdateVariableSolverData!(dfg, newvar) - # #For now spot check - # # @test solverDataDict(newvar) == solverDataDict(var1) - # @test getVariablePPEs(newvar) == getVariablePPEs(var1) - # - # # Delete :default and replace to see if new ones can be added - # delete!(getVariablePPEs(newvar), :default) - # getVariablePPEs(newvar)[:second] = MeanMaxPPE(:second, [15.0], [10.0], [5.0]) - # - # # Persist to the original variable. - # mergeUpdateVariableSolverData!(dfg, newvar) - # # At this point newvar will have only :second, and var1 should have both (it is the reference) - # @test symdiff(collect(keys(getVariablePPEs(var1))), [:default, :second]) == Symbol[] - # @test symdiff(collect(keys(getVariablePPEs(newvar))), [:second]) == Symbol[] - # # Get the source too. - # @test symdiff(collect(keys(getVariablePPEs(getVariable(dfg, :a)))), [:default, :second]) == Symbol[] - ## -end - function VSDTestBlock!(fg, v1) # "Variable Solver Data" # #### Variable Solver Data @@ -710,7 +582,7 @@ function VSDTestBlock!(fg, v1) # **State** # - `getSolveInProgress` - vnd = State{TestVariableType1}(; solveKey = :parametric) + vnd = State{TestVariableType1}(; label = :parametric) # vnd.val[1] = [0.0;] # vnd.bw[1] = [1.0;] @test addState!(fg, :a, vnd) == vnd @@ -743,12 +615,12 @@ function VSDTestBlock!(fg, v1) @test_throws LabelNotFoundError getState(fg, :a, :parametric) #FIXME copied from lower - @test getState(v1, :default) === v1.solverDataDict[:default] + @test getState(v1, :default) === v1.states[:default] # Add new VND of type ContinuousScalar to :x0 # Could also do State(ContinuousScalar()) - vnd = State{TestVariableType1}(; solveKey = :parametric) + vnd = State{TestVariableType1}(; label = :parametric) # vnd.val[1] = [0.0;] # vnd.bw[1] = [1.0;] @@ -768,8 +640,7 @@ function VSDTestBlock!(fg, v1) return nothing - #TODO solverDataDict() not deprecated - # @test getSolverDataDict(newvar) == getSolverDataDict(v1) + # @test refStates(newvar) == refStates(v1) # @test @test_deprecated mergeUpdateVariableSolverData!(fg, newvar) @@ -867,17 +738,7 @@ function DataEntriesTestBlock!(fg, v2) #delete from dfg @test deleteBlobentry!(fg, :a, :key2) == 1 @test listBlobentries(v1) == Symbol[] - deleteBlobentry!(fg, :b, :key2) - - # packed variable data entries - pacv = packVariable(v1) - @test addBlobentry!(pacv, de1) == de1 - @test hasBlobentry(pacv, :key1) - @test deepcopy(de1) == getBlobentry(pacv, :key1) - @test getBlobentries(pacv) == [deepcopy(de1)] - @test issetequal(listBlobentries(pacv), [:key1]) - # @test deleteBlobentry!(pacv, de1) == de1 - + return deleteBlobentry!(fg, :b, :key2) end function blobsStoresTestBlock!(fg) @@ -888,7 +749,7 @@ function blobsStoresTestBlock!(fg) crchash = 0xAAAA, origin = "origin1", description = "description1", - mimetype = "mimetype1", + mimetype = MIME("mimetype1"), ) de2 = Blobentry(; blobid = uuid4(), @@ -897,8 +758,8 @@ function blobsStoresTestBlock!(fg) crchash = 0xFFFF, origin = "origin2", description = "description2", - mimetype = "mimetype2", - timestamp = DFG.NanoDate("2020-08-12T12:00:00.000"), + mimetype = MIME("mimetype2"), + timestamp = DFG.TimeDateZone("2020-08-12T12:00:00.000Z"), ) de2_update = Blobentry(; blobid = uuid4(), @@ -907,8 +768,8 @@ function blobsStoresTestBlock!(fg) crchash = 0x0123, origin = "origin2", description = "description2", - mimetype = "mimetype2", - timestamp = DFG.NanoDate("2020-08-12T12:00:00.000"), + mimetype = MIME("mimetype2"), + timestamp = DFG.TimeDateZone("2020-08-12T12:00:00.000Z"), ) @test getLabel(de1) == de1.label @test getTimestamp(de1) == de1.timestamp @@ -1289,9 +1150,15 @@ function connectivityTestGraph( dfg = T(; graphLabel = :testGraph) vars = vcat( - map(n -> VARTYPE(Symbol("x$n"), State{TestVariableType1}()), 1:numNodesType1), map( - n -> VARTYPE(Symbol("x$(numNodesType1+n)"), State{TestVariableType2}()), + n -> VARTYPE(Symbol("x$n"), State{TestVariableType1}(; label = :default)), + 1:numNodesType1, + ), + map( + n -> VARTYPE( + Symbol("x$(numNodesType1+n)"), + State{TestVariableType2}(; label = :default), + ), 1:numNodesType2, ), ) @@ -1490,16 +1357,8 @@ function Summaries(testDFGAPI) # Check all fields are equal for all variables for v in ls(summaryGraph) for field in variableFields - if field != :variableTypeName - @test getproperty(getVariable(dfg, v), field) == - getproperty(getVariable(summaryGraph, v), field) - else - # Special case to check the symbol variableType is equal to the full variableType. - @test Symbol(typeof(getVariableType(getVariable(dfg, v)))) == - getVariableTypeName(getVariable(summaryGraph, v)) - @test getVariableType(getVariable(dfg, v)) == - getVariableType(getVariable(summaryGraph, v)) - end + @test getproperty(getVariable(dfg, v), field) == + getproperty(getVariable(summaryGraph, v), field) end end for f in lsf(summaryGraph) @@ -1523,10 +1382,10 @@ function ProducingDotFiles( dotdfg = testDFGAPI(; graphLabel = :testGraph) if v1 === nothing - v1 = VARTYPE(:a, State{TestVariableType1}()) + v1 = VARTYPE(:a, State{TestVariableType1}(; label = :default)) end if v2 === nothing - v2 = VARTYPE(:b, State{TestVariableType1}()) + v2 = VARTYPE(:b, State{TestVariableType1}(; label = :default)) end if f1 === nothing if (FACTYPE == FactorCompute) @@ -1679,18 +1538,16 @@ function FileDFGTestBlock(testDFGAPI; kwargs...) v4 = getVariable(dfg, :x4) vnd = getState(v4, :default) # set everything - vnd.BayesNetVertID = :outid - push!(vnd.BayesNetOutVertIDs, :id) + # vnd.BayesNetVertID = :outid + # push!(vnd.BayesNetOutVertIDs, :id) # vnd.bw[1] = [1.0;] - push!(vnd.dimIDs, 1) - vnd.dims = 1 - vnd.dontmargin = true - vnd.eliminated = true - vnd.infoPerCoord .= Float64[1.5;] + # vnd.dontmargin = true + # vnd.eliminated = true + vnd.observability .= Float64[1.5;] vnd.initialized = true - vnd.ismargin = true + vnd.marginalized = true push!(vnd.separator, :sep) - vnd.solvedCount = 2 + vnd.solves = 2 # vnd.val[1] = [2.0;] #update mergeVariable!(dfg, v4)