Remove old deprecations and implement new blob wrappers with load and save (#1173)

* rm v0.25 deprecations

* deprecate update[Blob|data] as obsolete

* Basic BlobWrappers ([save|load]Blob

* fix doxs

Author: Affie

docs/src/blob_ref.md

@@ -26,7 +26,7 @@ Pages = [
     "DataBlobs/services/BlobEntry.jl",
     "DataBlobs/services/BlobStores.jl",
     "DataBlobs/services/BlobPacking.jl",
-    "DataBlobs/services/HelpersDataWrapEntryBlob.jl",
+    "DataBlobs/services/BlobWrappers.jl",
 ]
 
 ```

src/DataBlobs/entities/BlobEntry.jl

@@ -2,7 +2,10 @@
 ##==============================================================================
 ## Blobentry
 ##==============================================================================
-
+#TODO think origin and buildSourceString should be deprecated, description can be used instead
+#TODO hash - maybe use both crc32c for fast error check and sha256 for strong integrity check
+#            stored seperately as crc and sha or as a tuple `hash::Tuple{Symbol, String}` where Symbol is :crc32c or :sha256
+#            or an enum with suppored hash types
 """
     $(TYPEDEF)
 
@@ -16,9 +19,7 @@ Base.@kwdef struct Blobentry
     """ Remotely assigned and globally unique identifier for the `Blobentry` itself (not the `.blobId`). """
     id::Union{UUID, Nothing} = nothing
     """ Machine friendly and globally unique identifier of the 'Blob', usually assigned from a common point in the system.  This can be used to guarantee unique retrieval of the large data blob. """
-    blobId::Union{UUID, Nothing} = uuid4() #Deprecated in v0.25 TODO  remove union, blobId is mandatory
-    """ TBD if Deprecated. Machine friendly and locally assigned identifier of the 'Blob'."""
-    originId::Union{UUID, Nothing} = blobId #Deprecated in v0.25 TODO remove union or remove originId completely
+    blobId::UUID = uuid4()
     """ Human friendly label of the `Blob` and also used as unique identifier per node on which a `Blobentry` is added.  E.g. do "LEFTCAM_1", "LEFTCAM_2", ... of you need to repeat a label on the same variable. """
     label::Symbol
     """ A hint about where the `Blob` itself might be stored.  Remember that a Blob may be duplicated over multiple blobstores. """
@@ -28,7 +29,7 @@ Base.@kwdef struct Blobentry
     """ Context from which a Blobentry=>Blob was first created. E.g. agent|graph|varlabel. """
     origin::String = ""
     """ number of bytes in blob as a string"""
-    size::Union{String, Nothing} = "-1" #TODO remove union -> size::String = "-1"
+    size::String = "-1"
     """ Additional information that can help a different user of the Blob. """
     description::String = ""
     """ MIME description describing the format of binary data in the `Blob`, e.g. 'image/png' or 'application/json; _type=CameraModel'. """
@@ -48,3 +49,42 @@ end
 StructTypes.StructType(::Type{Blobentry}) = StructTypes.UnorderedStruct()
 StructTypes.idproperty(::Type{Blobentry}) = :id
 StructTypes.omitempties(::Type{Blobentry}) = (:id,)
+
+function Blobentry(label::Symbol, blobstore = :default; kwargs...)
+    return Blobentry(; label, blobstore, kwargs...)
+end
+# construction helper from existing Blobentry for user overriding via kwargs
+function Blobentry(
+    entry::Blobentry;
+    id::Union{UUID, Nothing} = entry.id,
+    blobId::UUID = entry.blobId,
+    label::Symbol = entry.label,
+    blobstore::Symbol = entry.blobstore,
+    hash::String = entry.hash,
+    size::Union{String, Int, Nothing} = entry.size,
+    origin::String = entry.origin,
+    description::String = entry.description,
+    mimeType::String = entry.mimeType,
+    metadata::String = entry.metadata,
+    timestamp::ZonedDateTime = entry.timestamp,
+    createdTimestamp = entry.createdTimestamp,
+    lastUpdatedTimestamp = entry.lastUpdatedTimestamp,
+    _version = entry._version,
+)
+    return Blobentry(;
+        id,
+        blobId,
+        label,
+        blobstore,
+        hash,
+        origin,
+        size = string(size),
+        description,
+        mimeType,
+        metadata,
+        timestamp,
+        createdTimestamp,
+        lastUpdatedTimestamp,
+        _version,
+    )
+end

src/DataBlobs/services/BlobEntry.jl

@@ -1,7 +1,7 @@
 ##==============================================================================
 ## Blobentry - common
 ##==============================================================================
-
+#TODO think origin and buildSourceString should be deprecated, description can be used instead
 """
     $(SIGNATURES)
 Function to generate source string - agentLabel|graphLabel|varLabel
@@ -329,15 +329,16 @@ If the blob label `datalabel_1` already exists, then this function will return t
 """
 function incrDataLabelSuffix(
     dfg::AbstractDFG,
-    vla::Union{Symbol, <:AbstractString},
-    bllb::S;
+    vla::Symbol,
+    bllb::Union{Symbol, <:AbstractString};
     datalabel = Ref(""),
-) where {S <: Union{Symbol, <:AbstractString}}
+)
     count = 1
     hasund = false
     len = 0
     try
-        de, _ = getData(dfg, Symbol(vla), bllb)
+        de = getfirstBlobentry(dfg, vla; labelFilter = contains(string(bllb)))
+        isnothing(de) && return Symbol(bllb) # no match, return as is
         bllb = string(bllb)
         # bllb *= bllb[end] != '_' ? "_" : ""
         datalabel[] = string(de.label)
@@ -361,5 +362,5 @@ function incrDataLabelSuffix(
     end
     bllb *= string(count)
 
-    return S(bllb)
+    return Symbol(bllb)
 end

src/DataBlobs/services/BlobStores.jl

@@ -123,9 +123,6 @@ struct FolderStore{T} <: AbstractBlobstore{T}
     folder::String
 end
 
-#TODO added in v0.25 to avoid a breaking change in deserialization old DFGs, remove.
-StructTypes.StructType(::Type{<:FolderStore}) = StructTypes.OrderedStruct()
-
 function FolderStore(foldername::String; label::Symbol = :default, createfolder = true)
     storepath = joinpath(foldername, string(label))
     if createfolder && !isdir(storepath)
@@ -367,13 +364,6 @@ function addBlob!(store::RowBlobstore{T}, blobId::UUID, blob::T) where {T}
     return blobId
 end
 
-function updateBlob!(store::RowBlobstore{T}, blobId::UUID, blob::T) where {T}
-    if haskey(store.blobs, blobId)
-        @warn "Key '$blobId' doesn't exist."
-    end
-    return store.blobs[blobId] = RowBlob(blobId, blob)
-end
-
 function deleteBlob!(store::RowBlobstore, blobId::UUID)
     if !haskey(store.blobs, blobId)
         throw(IdNotFoundError("Blob", blobId))

src/DataBlobs/services/BlobWrappers.jl

@@ -0,0 +1,198 @@
+##==============================================================================
+## Blob + Blobentry CRUD interface
+##==============================================================================
+
+"""
+Convenience wrapper to load a Blob for a given variable and Blobentry label.
+
+$(METHODLIST)
+"""
+function loadBlob_Variable end
+
+"""
+Convenience wrapper to save a Blob to a Blobstore and a Blobentry to a variable.
+$(METHODLIST)
+"""
+function saveBlob_Variable! end
+
+"""
+Convenience wrapper to delete a Blob form a Blobstore and its Blobentry from a variable.
+$(METHODLIST)
+"""
+function deleteBlob_Variable! end
+
+"""
+Convenience wrapper to load a Blob for a given graph and Blobentry label.
+
+$(METHODLIST)
+"""
+function loadBlob_Graph end
+
+"""
+Convenience wrapper to save a Blob to a Blobstore and a Blobentry to a graph.
+
+$(METHODLIST)
+"""
+function saveBlob_Graph! end
+
+"""
+Convenience wrapper to delete a Blob from a Blobstore and its Blobentry from a graph.
+
+$(METHODLIST)
+"""
+function deleteBlob_Graph! end
+
+"""
+Convenience wrapper to load a Blob for a given agent and Blobentry label.
+
+$(METHODLIST)
+"""
+function loadBlob_Agent end
+
+"""
+Convenience wrapper to save a Blob to a Blobstore and a Blobentry to an agent.
+
+$(METHODLIST)
+"""
+function saveBlob_Agent! end
+
+"""
+Convenience wrapper to delete a Blob from a Blobstore and its Blobentry from an agent.
+
+$(METHODLIST)
+"""
+function deleteBlob_Agent! end
+
+function loadBlob_Variable(
+    dfg::AbstractDFG,
+    variable_label::Symbol,
+    entry_label::Symbol;
+    # hashfunction = sha256,
+    # checkhash::Bool = true,
+)
+    entry = getBlobentry(dfg, variable_label, entry_label)
+    blob = getBlob(dfg, entry)
+    # checkhash && assertHash(de, db; hashfunction)
+    return entry, blob
+end
+
+function saveBlob_Variable!(
+    dfg::AbstractDFG,
+    variable_label::Symbol,
+    blob::Vector{UInt8},
+    entry::Blobentry,
+)
+    addBlobentry!(dfg, variable_label, entry)
+    addBlob!(dfg, entry, blob)
+    return entry
+end
+
+function saveBlob_Variable!(
+    dfg::AbstractDFG,
+    variable_label::Symbol,
+    blob::Vector{UInt8},
+    entry_label::Symbol,
+    blobstore::Symbol = :default;
+    blobentry_kwargs...,
+)
+    entry = Blobentry(entry_label, blobstore; blobentry_kwargs...)
+    return saveBlob_Variable!(dfg, variable_label, blob, entry)
+end
+
+function deleteBlob_Variable!(dfg::AbstractDFG, variable_label::Symbol, entry_label::Symbol)
+    entry = getBlobentry(dfg, variable_label, entry_label)
+    deleteBlobentry!(dfg, variable_label, entry_label)
+    deleteBlob!(dfg, entry)
+    return 2
+end
+
+function loadBlob_Graph(dfg::AbstractDFG, entry_label::Symbol;)
+    entry = getGraphBlobentry(dfg, entry_label)
+    blob = getBlob(dfg, entry)
+    return entry, blob
+end
+
+function saveBlob_Graph!(dfg::AbstractDFG, blob::Vector{UInt8}, entry::Blobentry)
+    addGraphBlobentry!(dfg, entry)
+    addBlob!(dfg, entry, blob)
+    return entry
+end
+
+function saveBlob_Graph!(
+    dfg::AbstractDFG,
+    blob::Vector{UInt8},
+    entry_label::Symbol,
+    blobstore::Symbol = :default;
+    blobentry_kwargs...,
+)
+    entry = Blobentry(entry_label, blobstore; blobentry_kwargs...)
+    return saveBlob_Graph!(dfg, blob, entry)
+end
+
+function deleteBlob_Graph!(dfg::AbstractDFG, entry_label::Symbol)
+    entry = getGraphBlobentry(dfg, entry_label)
+    deleteGraphBlobentry!(dfg, entry_label)
+    deleteBlob!(dfg, entry)
+    return 2
+end
+
+function loadBlob_Agent(dfg::AbstractDFG, entry_label::Symbol;)
+    entry = getAgentBlobentry(dfg, entry_label)
+    blob = getBlob(dfg, entry)
+    return entry, blob
+end
+
+function saveBlob_Agent!(dfg::AbstractDFG, blob::Vector{UInt8}, entry::Blobentry)
+    addAgentBlobentry!(dfg, entry)
+    addBlob!(dfg, entry, blob)
+    return entry
+end
+
+function saveBlob_Agent!(
+    dfg::AbstractDFG,
+    blob::Vector{UInt8},
+    entry_label::Symbol,
+    blobstore::Symbol = :default;
+    blobentry_kwargs...,
+)
+    entry = Blobentry(entry_label, blobstore; blobentry_kwargs...)
+    return saveBlob_Agent!(dfg, blob, entry)
+end
+
+function deleteBlob_Agent!(dfg::AbstractDFG, entry_label::Symbol)
+    entry = getAgentBlobentry(dfg, entry_label)
+    deleteAgentBlobentry!(dfg, entry_label)
+    deleteBlob!(dfg, entry)
+    return 2
+end
+
+function saveImage_Variable!(
+    dfg::AbstractDFG,
+    variable_label::Symbol,
+    img::AbstractMatrix,
+    entry_label::Symbol,
+    blobstore::Symbol = :default;
+    entry_kwargs...,
+)
+    mimeType = get(entry_kwargs, :mimeType, MIME("image/png"))
+    format = _MIMETypes[mimeType]
+
+    blob, mimeType = packBlob(format, img)
+
+    size = string(length(blob))
+    entry = Blobentry(
+        entry_label,
+        blobstore;
+        blobId = uuid4(),
+        entry_kwargs...,
+        size,
+        mimeType = string(mimeType),
+    )
+
+    return saveBlob_Variable!(dfg, variable_label, blob, entry)
+end
+
+function loadImage_Variable(dfg::AbstractDFG, variable_label::Symbol, entry_label::Symbol)
+    entry, blob = loadBlob_Variable(dfg, variable_label, entry_label)
+    return entry, unpackBlob(entry, blob)
+end