First draft on cache synchronization

src/LRUCaching.jl

@@ -19,9 +19,9 @@ export AbstractCache,
     empty!,
     @diskcache,
     @memcache,
-    @syncache,
-    @persist,
-    @empty
+    @syncache!,
+    @persist!,
+    @empty!
 
 include("memcache.jl")
 include("diskcache.jl")

src/diskcache.jl

@@ -45,7 +45,6 @@ end
 # 	julia> fooc = @diskcache foo # now `fooc` is the cached version of `foo`
 macro diskcache(symb::Symbol, filename::String=
                 _generate_cache_filename(string(symb)))
-    @assert isdefined(Main, symb)
     _name = String(symb)
     ex = quote
         try
@@ -72,7 +71,6 @@ macro diskcache(expr::Expr, filename::String=
 	_type = eval(_typesymbol)
 	_name = String(_symb)
 
-	@assert isdefined(Main, _symb)  # check that symbol exists
 	try
         @assert _type isa Type
 	catch  # it may be a variable containing a type

src/memcache.jl

@@ -21,7 +21,7 @@ MemoryCache(f::T where T<:Function;
     if _hash in keys(mc.cache)
         out = mc.cache[_hash]
     else
-        @info "Hash miss, caching hash=$_hash..."
+        @debug "Hash miss, caching hash=$_hash..."
         out = mc.func(args...; kwargs...)
         mc.cache[_hash] = out
     end
@@ -42,7 +42,6 @@ end
 # 	julia> foo(x) = x+1
 # 	julia> fooc = @memcache foo # now `fooc` is the cached version of `foo`
 macro memcache(symb::Symbol)
-    @assert isdefined(Main, symb)
     _name = String(symb)
     ex = quote
         try
@@ -66,7 +65,6 @@ macro memcache(expr::Expr)
     _type = eval(_typesymbol)
     _name = String(_symb)
 
-    @assert isdefined(Main, _symb)  # check that symbol exists
     try
         @assert _type isa Type
     catch  # it may be a variable containing a type

src/utils.jl

@@ -1,55 +1,101 @@
-# Additional constructors
+# Convert MemoryCache to DiskCache (constructor)
 DiskCache(mc::MemoryCache{T, I, O};
 		  filename::String = _generate_cache_filename(mc.name)) where {T, I, O} =
     DiskCache(filename, deepcopy(mc), Dict{I, Tuple{Int, Int}}())
 
 
+# Convert DiskCache to MemoryCache (constructor)
 MemoryCache(dc::DiskCache) = deepcopy(dc.memcache)
 
 
+# Function that checks the consistency of the disk cache and `offset`
+# field value of the DiskCache object
+function _check_disk_cache(filename::String, offsets::D where D<:Dict,
+                           input_type::Type, output_type::Type)::Bool
+    fid = open(filename, "r")
+    try
+        I = deserialize(fid)
+		@assert I == input_type
+        O = deserialize(fid)
+		@assert O == output_type
+        for (_, (prevpos, newpos)) in offsets
+            seek(fid, prevpos)
+            out = deserialize(fid)
+        end
+        return true
+    catch excep
+        close(fid)
+        return false
+    end
+end
+
+
 # `with` parameter behavior:
 #   "both" - memory and disk concents are combined, memory values update disk
 #   "disk" - memory cache contents are updated with disk ones
 #   "memory" - disk cache contents are updated with memory ones
 function syncache!(dc::DiskCache{T, I, O};
-                   with::String="both",
-                   mode::String="inclusive") where {T, I, O}
+                   with::String="both") where {T, I, O}
     # Check keyword argument values, correct unknown values
     _default_with = "both"
-    _default_mode = "inclusive"
+    noff = length(dc.offsets)
     !(with in ["disk", "memory", "both"]) && begin
         @warn "Unrecognized value with=$with, defaulting to $_default_with."
         with = _default_with
     end
-    !(mode in ["inclusive", "exclusive"]) && begin
-        @warn "Unrecognized value mode=$with, defaulting to $_default_mode."
-        with = _default_mode
-    end
 
     # Cache synchronization
     if !isfile(dc.filename)
         if with == "both" || with == "memory"
-            noff = length(dc.offsets)
-            noff != 0 && @warn "Missing cache file, $noff existing offsets will be deleted."
+            noff != 0 && @warn "Missing cache file, will write memory cache to disk."
             persist!(dc)
         else
+            @warn "Missing cache file, will delete all cache."
             empty!(dc)
         end
     else
         cache_ok = _check_disk_cache(dc.filename, dc.offsets, I, O)
-        # TODO(Corneliu) Add warnings here
-        !cache_ok && with != "disk" && persist!(dc)
-        !cache_ok && with == "disk" && empty!(dc, empty_disk=true)
-        # At this point, the `offsets` dictionary should reflect
-        # the structure of the file pointed at by the `filename` field
-        mode = ifelse(with == "both" || with == "memory", "w+", "r")
+        if !cache_ok && with != "disk"
+            @warn "Inconsistent cache, overwriting disk contents."
+            persist!(dc)
+        elseif !cache_ok && with == "disk"
+            @warn "Inconsistent cache, will delete all cache."
+            empty!(dc, empty_disk=true)
+        else  # cache_ok
+            # At this point, the `offsets` dictionary should reflect
+            # the structure of the file pointed at by the `filename` field
+            mode = ifelse(with == "both" || with == "memory", "a+", "r")
+            memonly = setdiff(keys(dc.memcache.cache), keys(dc.offsets))
+            diskonly = setdiff(keys(dc.offsets), keys(dc.memcache.cache))
+            fid = open(dc.filename, mode)
+            # Load from disk to memory
+            if with != "memory"
+                for hash in diskonly
+                    pos = dc.offsets[hash][1]
+                    seek(fid, pos)
+                    datum  = deserialize(fid)
+                    push!(dc.memcache.cache, hash=>datum)
+                end
+            end
+            # Write memory to disk and update offsets
+            if with != "disk"
+                for hash in memonly
+                    datum = dc.memcache.cache[hash]
+                    prevpos = position(fid)
+                    serialize(fid, datum);
+                    newpos = position(fid)
+                    push!(dc.offsets, hash=>(prevpos, newpos))
+                end
+            end
+            close(fid)
+        end
     end
     return dc
 end
 
 
-macro syncache!(symb::Symbol, with::String="both", mode::String="inclusive")
-    return esc(:(syncache!($symb, with=$with, mode=$mode)))
+macro syncache!(symb::Symbol, with::String="both")
+    return esc(:(syncache!($symb, with=$with)))
 end
 
 
@@ -88,7 +134,7 @@ function persist!(dc::T; filename::String=dc.filename) where T<:DiskCache
 end
 
 
-macro persist(symb::Symbol, filename::String...)
+macro persist!(symb::Symbol, filename::String...)
     if isempty(filename)
         return esc(:(persist!($symb)))
     else
@@ -115,6 +161,6 @@ function empty!(dc::DiskCache; empty_disk::Bool=false)
 end
 
 
-macro empty(symb::Symbol, empty_disk::Bool=false)
+macro empty!(symb::Symbol, empty_disk::Bool=false)
     return esc(:(empty!($symb, empty_disk=$empty_disk)))
 end

test/runtests.jl

@@ -7,19 +7,13 @@ _a_float = rand()
 _a_float_2 = rand()
 _an_int = rand(Int)
 
-N = 3
-
-# Functions
-function foo(x)
-	return x
-end
-
-function bar(x; y=1)
-	return x+y
-end
 
 # Test MemoryCache, @memcahe
 @testset "MemoryCache" begin
+    # Define functions
+    foo(x) = x
+    bar(x; y=1) = x + y
+
     # Test caching of simple functions
     foo_c1 = LRUCaching.MemoryCache(foo)
     @test typeof(foo_c1) <: LRUCaching.AbstractCache
@@ -58,6 +52,10 @@ end
 
 # Test DiskCache, @diskcache
 @testset "DiskCache" begin
+    # Define functions
+    foo(x) = x
+    bar(x; y=1) = x + y
+
     # Test caching of simple functions
     foo_c1 = LRUCaching.DiskCache(foo)
     @test typeof(foo_c1) <: LRUCaching.AbstractCache
@@ -102,7 +100,7 @@ end
 
 # Test functionality contained in the utils
 @testset "Conversion constructors" begin
-    _tmpfile = "_tmpfile.bin"
+    # Define functions
     foo(x) = x+1
     bar(x) = x-1
 
@@ -111,11 +109,13 @@ end
     dc = @diskcache foo
 
     # Construct cache objects using conversions
+    _tmpfile = "_tmpfile.bin"
     mc_t = MemoryCache(dc)
     dc_t1 = DiskCache(mc)
     dc_t2 = DiskCache(mc, filename=_tmpfile)
 
     # Fill the cache
+    N = 3
     for i in 1:N
         mc_t(rand())
         dc_t1(rand())
@@ -141,51 +141,94 @@ end
 end
 
 
-# TODO(Corneliu): test synccache!, @synccache
-@testset "syncache!, @syncache" begin
+# syncache!, @syncache!
+@testset "syncache!, @syncache!" begin
+    # Define functions
+    foo(x) = x+1  # define function
+
+    N1 = 5
+    fc = @diskcache foo "somefile.bin"  # make a DiskCache object
+    [fc(i) for i in 1:N1]  # populate the memorycache
+    @persist! fc  # write to disk the cache
+    @empty! fc  # delete the memory cache
+    @test length(fc.memcache.cache) == 0
+    @syncache! fc "disk"  # load cache from disk
+    @test isfile(fc.filename)
+    @test length(fc.memcache.cache) == N1
+    @empty! fc  # 5 entries on disk, 0 in memory
+
+    N2 = 3
+    [fc(-i) for i in 1:N2]  # populate the memory cache
+    @syncache! fc "memory"  # write memory cache to disk
+    @test length(fc.memcache.cache) == N2
+    @empty! fc
+    @test length(fc.memcache.cache) == 0
+    @syncache! fc "disk"    # load cache from disk
+    @test length(fc.memcache.cache) == N1 + N2
+    @empty! fc true  # remove everything
+
+    [fc(i) for i in 1:N1]  # populate the memorycache
+    @syncache! fc "memory"  # write to disk
+    @empty! fc
+    [fc(-i) for i in 1:N1]  # populate the memorycache
+    @syncache! fc "both"     # sync both memory and disk
+    @test length(fc.offsets) == 2*N1
+    @test length(fc.memcache.cache) == 2*N1
+    @empty! fc true
+    @test !isfile(fc.filename)
 end
 
-# empty!, @empty
-@testset "empty!, @empty" begin
+
+# empty!, @empty!
+@testset "empty!, @empty!" begin
+    # Define functions
+    foo(x) = x
+    bar(x; y=1) = x + y
+
     mc = @memcache foo; mc(1)
     @test length(mc.cache) == 1
-    @empty mc
+    @empty! mc
     @test isempty(mc.cache)
 
     dc = @diskcache foo; dc(1)
     @test length(dc.memcache.cache) == 1
-    @persist dc "somefile.bin"
+    @persist! dc "somefile.bin"
     @test length(dc.offsets) == 1
     @test isfile("somefile.bin")
-    @empty dc true  # remove offsets and file
+    @empty! dc true  # remove offsets and file
     @test isempty(dc.memcache.cache)
     @test !isfile("somefile.bin")
     @test isempty(dc.offsets)
 end
 
-# persist!, @persist
-@testset "persist!, @persist" begin
+
+# persist!, @persist!
+@testset "persist!, @persist!" begin
+    # Define functions
+    foo(x) = x
+
     mc = @memcache foo
     dc = @diskcache foo "somefile.bin"
     @test dc.filename == abspath("somefile.bin")
     @test isempty(dc.offsets)
+    N = 3
     for i in 1:N dc(i); mc(i) end  # add N entries
 
-    _path, _offsets = @persist mc "memfile.bin"
+    _path, _offsets = @persist! mc "memfile.bin"
     @test isabspath(_path)
     @test typeof(_offsets) <: Dict{<:Unsigned, <:Tuple{Int, Int}}
     @test length(_offsets) == N
     @test isfile("memfile.bin")
     rm("memfile.bin")
 
-    @persist dc
+    @persist! dc
     @test length(dc.offsets) == N
     @test isfile("somefile.bin")
     @test dc.filename == abspath("somefile.bin")
     buf = open(read, "somefile.bin")
     rm("somefile.bin")
 
-    @persist dc "some_other_file.bin"
+    @persist! dc "some_other_file.bin"
     @test length(dc.offsets) == N
     @test isfile("some_other_file.bin")
     @test dc.filename == abspath("some_other_file.bin")