From a8f078deb7f5361f6dc4d653afb4bdfde691a40b Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Wed, 2 Oct 2019 13:34:08 +0200 Subject: [PATCH] Add BigData to DFGVariable --- src/BigData.jl | 139 +++++++++++++++++++++++++++++++++ src/DistributedFactorGraphs.jl | 2 + src/entities/AbstractDFG.jl | 6 ++ src/entities/DFGVariable.jl | 19 +++-- test/interfaceTests.jl | 44 +++++++++++ 5 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 src/BigData.jl diff --git a/src/BigData.jl b/src/BigData.jl new file mode 100644 index 00000000..0c1f3e41 --- /dev/null +++ b/src/BigData.jl @@ -0,0 +1,139 @@ + +# TODO move to ... + + +# """ +# $(TYPEDEF) +# Abstract parent struct for big data entry. +# """ +# abstract type AbstractBigDataEntry end + +# --- AbstractBigData Interfaces --- +# fields to implement: +# - key::Symbol +# available methods: +# - addBigDataEntry! +# - getBigDataEntry +# - updateBigDataEntry! +# - deleteBigDataEntry! +# - getBigDataEntries +# - getBigDataKeys + +export addBigDataEntry!, + getBigDataEntry, + updateBigDataEntry!, + deleteBigDataEntry!, + getBigDataEntries, + getBigDataKeys, + MongodbBigDataEntry, + FileBigDataEntry + +# Methods + +#TODO should this return Bool or the modified Variable? +""" + $(SIGNATURES) +Add Big Data Entry to a DFG variable +""" +function addBigDataEntry!(var::AbstractDFGVariable, bde::AbstractBigDataEntry)::Bool + haskey(var.bigData,bde.key) && @warn "$(bde.key) already exists in variable, overwriting!" + var.bigData[bde.key] = bde + return true +end + +""" + $(SIGNATURES) +Add Big Data Entry to distrubuted factor graph. +Should be extended if DFG variable is not returned by by reference. +""" +function addBigDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractBigDataEntry)::Bool + return addBigDataEntry!(getVariable(dfg, label), bde) +end + +""" + $(SIGNATURES) +Get big data entry +""" +function getBigDataEntry(var::AbstractDFGVariable, key::Symbol)::AbstractBigDataEntry + return var.bigData[key] +end +function getBigDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)::AbstractBigDataEntry + return getBigDataEntry(getVariable(dfg, label), key) +end + +""" + $(SIGNATURES) +Update big data entry +""" +function updateBigDataEntry!(var::AbstractDFGVariable, bde::AbstractBigDataEntry)::Bool#TODO should this return Bool? + !haskey(var.bigData,bde.key) && (@error "$(bde.key) does not exist in variable!"; return false) + var.bigData[bde.key] = bde + return true +end +function updateBigDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractBigDataEntry)::Bool + updateBigDataEntry!(getVariable(dfg, label), bde) +end + +""" + $(SIGNATURES) +Delete big data entry +""" +function deleteBigDataEntry!(var::AbstractDFGVariable, key::Symbol)::AbstractBigDataEntry #users responsibility to delete big data in db before deleting entry + bde = getBigDataEntry(var, key) + delete!(var.bigData, key) + return bde +end + +function deleteBigDataEntry!(dfg::AbstractDFG, label::Symbol, key::Symbol)::AbstractBigDataEntry #users responsibility to delete big data in db before deleting entry + deleteBigDataEntry!(getVariable(dfg, label), key) +end + +""" + $(SIGNATURES) +Get big data entries, Vector{AbstractBigDataEntry} +""" +function getBigDataEntries(var::AbstractDFGVariable)::Vector{AbstractBigDataEntry} + #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}? + collect(values(var.bigData)) +end +function getBigDataEntries(dfg::AbstractDFG, label::Symbol)::Vector{AbstractBigDataEntry} + #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}? + getBigDataEntries(getVariable(dfg, label)) +end + + +""" + $(SIGNATURES) +getBigDataKeys +""" +function getBigDataKeys(var::AbstractDFGVariable)::Vector{Symbol} + collect(keys(var.bigData)) +end +function getBigDataKeys(dfg::AbstractDFG, label::Symbol)::Vector{Symbol} + getBigDataKeys(getVariable(dfg, label)) +end + + + +# Types <: AbstractBigDataEntry +""" + $(TYPEDEF) +BigDataEntry in MongoDB. +""" +struct MongodbBigDataEntry <: AbstractBigDataEntry + key::Symbol + oid::NTuple{12, UInt8} #mongodb object id + #maybe other fields such as: + #flags::Bool ready, valid, locked, permissions + #MIMEType::Symbol +end + + +""" + $(TYPEDEF) +BigDataEntry in a file. +""" +struct FileBigDataEntry <: AbstractBigDataEntry + key::Symbol + filename::String +end diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 7f6605c8..f79a5bd2 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -48,6 +48,8 @@ include("services/AbstractDFG.jl") include("services/DFGVariable.jl") include("services/DFGFactor.jl") +include("BigData.jl") + # Include the Graphs.jl API. include("GraphsDFG/GraphsDFG.jl") diff --git a/src/entities/AbstractDFG.jl b/src/entities/AbstractDFG.jl index e7d1118f..5027c586 100644 --- a/src/entities/AbstractDFG.jl +++ b/src/entities/AbstractDFG.jl @@ -39,3 +39,9 @@ Empty structure for solver parameters. """ mutable struct NoSolverParams <: AbstractParams end + +""" + $(TYPEDEF) +Abstract parent struct for big data entry. +""" +abstract type AbstractBigDataEntry end diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index 02d83906..3c7e7a17 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -80,8 +80,9 @@ struct VariableEstimate end """ - $(SIGNATURES) -Fundamental structure for a DFG variable. + $(TYPEDEF) +Fundamental structure for a DFG variable with fields: +$(TYPEDFIELDS) """ mutable struct DFGVariable <: AbstractDFGVariable label::Symbol @@ -90,14 +91,22 @@ mutable struct DFGVariable <: AbstractDFGVariable estimateDict::Dict{Symbol, Dict{Symbol, VariableEstimate}} solverDataDict::Dict{Symbol, VariableNodeData} smallData::Dict{String, String} - bigData::Any + bigData::Dict{Symbol, AbstractBigDataEntry} ready::Int backendset::Int _internalId::Int64 - DFGVariable(label::Symbol, _internalId::Int64) = new(label, now(), Symbol[], Dict{Symbol, Dict{Symbol, VariableEstimate}}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), nothing, 0, 0, _internalId) - DFGVariable(label::Symbol) = new(label, now(), Symbol[], Dict{Symbol, VariableEstimate}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), nothing, 0, 0, 0) end +""" + $SIGNATURES +DFGVariable constructors. +""" +DFGVariable(label::Symbol, _internalId::Int64) = + DFGVariable(label, now(), Symbol[], Dict{Symbol, Dict{Symbol, VariableEstimate}}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), Dict{Symbol,AbstractBigDataEntry}(), 0, 0, _internalId) + +DFGVariable(label::Symbol) = + DFGVariable(label, now(), Symbol[], Dict{Symbol, VariableEstimate}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), Dict{Symbol,AbstractBigDataEntry}(), 0, 0, 0) + # Accessors label(v::DFGVariable) = v.label timestamp(v::DFGVariable) = v.timestamp diff --git a/test/interfaceTests.jl b/test/interfaceTests.jl index 9dbf6b21..2b277e5f 100644 --- a/test/interfaceTests.jl +++ b/test/interfaceTests.jl @@ -118,6 +118,50 @@ end end +@testset "BigData" begin + oid = zeros(UInt8,12); oid[12] = 0x01 + de1 = MongodbBigDataEntry(:key1, NTuple{12,UInt8}(oid)) + + oid = zeros(UInt8,12); oid[12] = 0x02 + de2 = MongodbBigDataEntry(:key2, NTuple{12,UInt8}(oid)) + + oid = zeros(UInt8,12); oid[12] = 0x03 + de2_update = MongodbBigDataEntry(:key2, NTuple{12,UInt8}(oid)) + + #add + v1 = getVariable(dfg, :a) + @test addBigDataEntry!(v1, de1) + @test addBigDataEntry!(dfg, :a, de2) + @test addBigDataEntry!(v1, de1) + + #get + @test deepcopy(de1) == getBigDataEntry(v1, :key1) + @test deepcopy(de2) == getBigDataEntry(dfg, :a, :key2) + @test_throws Any getBigDataEntry(v2, :key1) + @test_throws Any getBigDataEntry(dfg, :b, :key1) + + #update + @test updateBigDataEntry!(dfg, :a, de2_update) + @test deepcopy(de2_update) == getBigDataEntry(dfg, :a, :key2) + @test !updateBigDataEntry!(dfg, :b, de2_update) + + #list + entries = getBigDataEntries(dfg, :a) + @test length(entries) == 2 + @test symdiff(map(e->e.key, entries), [:key1, :key2]) == Symbol[] + @test length(getBigDataEntries(dfg, :b)) == 0 + + @test symdiff(getBigDataKeys(dfg, :a), [:key1, :key2]) == Symbol[] + @test getBigDataKeys(dfg, :b) == Symbol[] + + #delete + @test deepcopy(de1) == deleteBigDataEntry!(v1, :key1) + @test getBigDataKeys(v1) == Symbol[:key2] + #delete from dfg + @test deepcopy(de2_update) == deleteBigDataEntry!(dfg, :a, :key2) + @test getBigDataKeys(v1) == Symbol[] +end + @testset "Updating Nodes" begin global dfg #get the variable