@@ -26,4 +26,14 @@ D = (blocksize-5)/16 key file: -[key\0,key\0,key\0] \ No newline at end of file +[key\0,key\0,key\0] + +keys allocation + +l = 2**4 + +m = 2**64 + + + +header size = 85 + (√blocksize - 4)*8 \ No newline at end of file BTREEFORMAT @@ -18,7 +18,11 @@ task :default => [:build_deps] do # end end -task :clean do +task :clean_test do + sh "rm -rf etest/log/*" +end + +task :clean => [:clean_test] do sh "rm -f ebin/*.beam" sh "rm -f etest/*.beam" end Rakefile @@ -245,19 +245,19 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%==================================================================== delete(KeyHash, Key, Parent, Node = #node{children=Children,keys=Keys}, Tree) -> - % error_logger:info_msg("delete key ~p keyhash ~p from node ~p~n", [Key, KeyHash, Node]), + error_logger:info_msg("delete key ~p keyhash ~p from node ~p~n", [Key, KeyHash, Node]), {WhatItDo, DeleteNode} = case find_child_adj(KeyHash, Keys, Children) of {FoundKey, {{LeftHash, LeftPointer}, {RightHash, RightPointer}}} -> LeftNode = dmtree:read(LeftPointer, Tree), RightNode = dmtree:read(RightPointer, Tree), - % ?infoFmt("delete_merge foundkey ~p~nLeftPointer ~p~nrightpointer ~p~nparent ~p~nnode ~p~nleftnode ~p~nrightnode ~p~n", [FoundKey, LeftPointer, RightPointer, Parent, Node, LeftNode, RightNode]), + ?infoFmt("delete_merge foundkey ~p~nLeftPointer ~p~nrightpointer ~p~nparent ~p~nnode ~p~nleftnode ~p~nrightnode ~p~n", [FoundKey, LeftPointer, RightPointer, Parent, Node, LeftNode, RightNode]), delete_merge(FoundKey, dmtree:d(Tree), Parent, Node, LeftNode, RightNode, Tree); {FoundKey, {{LeftHash, LeftPointer}, undefined}} -> {nothing, dmtree:read(LeftPointer, Tree)} end, - % ?infoMsg("recursing into delete~n"), + ?infoMsg("recursing into delete~n"), ReturnNode = delete(KeyHash, Key, Node, DeleteNode, Tree), - % ?infoFmt("reduced from delete ~p~n", [ReturnNode]), + ?infoFmt("reduced from delete ~p~n", [ReturnNode]), Eqls = ref_equals(ReturnNode, Node), case WhatItDo of wamp_wamp -> ReturnNode; @@ -266,22 +266,22 @@ delete(KeyHash, Key, Parent, Node = #node{children=Children,keys=Keys}, Tree) -> end; delete(KeyHash, Key, Parent, Leaf = #leaf{values=Values}, Tree) -> - % error_logger:info_msg("delete key ~p keyhash ~p from ~p~n", [Key, KeyHash, Leaf]), + error_logger:info_msg("delete key ~p keyhash ~p from ~p~n", [Key, KeyHash, Leaf]), case lists:keytake(KeyHash, 1, Values) of {value, {KeyHash,Pointer,ValHash}, NewValues} -> NewLeaf = Leaf#leaf{values=NewValues,m=length(NewValues)}, - % ?infoFmt("new leaf ~p~n", [NewLeaf]), + ?infoFmt("new leaf ~p~n", [NewLeaf]), dmtree:delete_key(Pointer, Key, Tree), dmtree:write(NewLeaf, Tree); false -> - % ?infoFmt("couldnt find ~p in ~p~n", [KeyHash, Leaf]), + ?infoFmt("couldnt find ~p in ~p~n", [KeyHash, Leaf]), Leaf end. update_hash(Hash, Pointer, Node = #node{children=Children}, Tree) -> - % ?infoFmt("updating hash,ptr ~p for ~p~n", [{Hash,Pointer}, Node]), + ?infoFmt("updating hash,ptr ~p for ~p~n", [{Hash,Pointer}, Node]), NewNode = Node#node{children=lists:keyreplace(Pointer, 2, Children, {Hash,Pointer})}, - % ?infoFmt("updated node ~p~n", [NewNode]), + ?infoFmt("updated node ~p~n", [NewNode]), dmtree:write(NewNode, Tree). % delete_merge(FoundKey, @@ -295,7 +295,7 @@ delete_merge(FoundKey, D, LeftLeaf = #leaf{values=LeftValues,m=LeftM}, RightLeaf = #leaf{values=RightValues,m=RightM}, Tree) when (LeftM+RightM) =< D, PM == 1 -> - % ?infoMsg("Replacing root merging leaves~n"), + ?infoMsg("Replacing root merging leaves~n"), dmtree:delete(Root#node.offset, Tree), dmtree:delete(RightLeaf#leaf.offset, Tree), NewLeaf = dmtree:write(LeftLeaf#leaf{m=LeftM+RightM,values=LeftValues++RightValues}, Tree), @@ -308,12 +308,12 @@ delete_merge(FoundKey, D, LeftLeaf = #leaf{values=LeftValues,m=LeftM}, RightLeaf = #leaf{values=RightValues,m=RightM}, Tree) when (LeftM+RightM) =< D, length(PKeys) == 1 -> - % ?infoMsg("Replacing node merging leaves~n"), + ?infoMsg("Replacing node merging leaves~n"), dmtree:delete(Parent#node.offset, Tree), dmtree:delete(RightLeaf#leaf.offset, Tree), NewLeaf = dmtree:write(LeftLeaf#leaf{m=LeftM+RightM,values=LeftValues++RightValues}, Tree), NP = dmtree:write(SuperParent#node{children=lists:keyreplace(offset(Parent), 2, PChildren, {hash(NewLeaf),offset(NewLeaf)})}, Tree), - % ?infoFmt("replaced ~p in super parent ~p~n", [{hash(Parent),offset(Parent)}, NP]), + ?infoFmt("replaced ~p in super parent ~p~n", [{hash(Parent),offset(Parent)}, NP]), {wamp_wamp, NewLeaf}; delete_merge(FoundKey, D, @@ -323,16 +323,16 @@ delete_merge(FoundKey, D, RightLeaf = #leaf{values=RightValues,m=RightM}, Tree) when (LeftM+RightM) =< D -> %we can merge within reqs gogogo - % ?infoMsg("Merging leaves~n"), + ?infoMsg("Merging leaves~n"), N = if FoundKey == last -> length(PKeys); true -> lib_misc:position(FoundKey, PKeys) end, - % ?infoFmt("FoundKey ~p~nPKeys ~p~nPChildren ~p~nN ~p~nLeftM ~p~nRightM ~p~nD ~p~n", [FoundKey, PKeys, PChildren, N, LeftM, RightM, D]), + ?infoFmt("FoundKey ~p~nPKeys ~p~nPChildren ~p~nN ~p~nLeftM ~p~nRightM ~p~nD ~p~n", [FoundKey, PKeys, PChildren, N, LeftM, RightM, D]), NP = dmtree:write(remove_nth(Parent, N), Tree), dmtree:write(LeftLeaf#leaf{m=LeftM+RightM,values=LeftValues++RightValues}, Tree), dmtree:delete(RightLeaf#leaf.offset, Tree), - % ?infoFmt("new parent: ~p~n", [NP]), + ?infoFmt("new parent: ~p~n", [NP]), { merge, NP}; %merging nodes @@ -342,7 +342,7 @@ delete_merge(FoundKey, D, LeftNode = #node{m=LeftM}, RightNode = #node{m=RightM}, Tree) when (LeftM+RightM) < D, length(PKeys) == 1 -> - % ?infoMsg("replacing root merging nodes~n"), + ?infoMsg("replacing root merging nodes~n"), NC = dmtree:write(merge_nodes(FoundKey, PKeys, LeftNode, RightNode), Tree), dmtree:delete(offset(Root), Tree), dmtree:delete(offset(RightNode), Tree), @@ -355,14 +355,14 @@ delete_merge(FoundKey, D, LeftNode = #node{m=LeftM,keys=LeftKeys,children=LeftChildren}, RightNode = #node{m=RightM,keys=RightKeys,children=RightChildren}, Tree) when (LeftM+RightM) < D, length(PKeys) == 1 -> - % ?infoMsg("Replacing node merging nodes~n"), + ?infoMsg("Replacing node merging nodes~n"), dmtree:delete(offset(RightNode), Tree), dmtree:delete(offset(Parent), Tree), ParentPointer = offset(Parent), ParentHash = hash(Parent), NN = merge_nodes(FoundKey, PKeys, LeftNode, RightNode), NP = dmtree:write(SuperParent#node{children=lists:keyreplace(offset(Parent), 2, PChildren, {hash(NN),offset(NN)})}, Tree), - % ?infoFmt("NN: ~p~n", [NN]), + ?infoFmt("NN: ~p~n", [NN]), NewNode = dmtree:write(NN, Tree), {wamp_wamp, NewNode}; @@ -372,26 +372,26 @@ delete_merge(FoundKey, D, LeftNode = #node{keys=LeftKeys,children=LeftChildren,m=LM}, RightNode = #node{keys=RightKeys,children=RightChildren,m=RM}, Tree) when (LM+RM) < D -> - % ?infoMsg("merging nodes~n"), + ?infoMsg("merging nodes~n"), N = if FoundKey == last -> length(PKeys) -1; true -> lib_misc:position(FoundKey, PKeys) end, - % ?infoFmt("FoundKey ~p~nPKeys ~p~nPChildren ~p~nN ~p~nLeftM ~p~nRightM ~p~nD ~p~n", [FoundKey, PKeys, PChildren, N, LM, RM, D]), + ?infoFmt("FoundKey ~p~nPKeys ~p~nPChildren ~p~nN ~p~nLeftM ~p~nRightM ~p~nD ~p~n", [FoundKey, PKeys, PChildren, N, LM, RM, D]), NP = dmtree:write(remove_nth(Parent, N), Tree), NC = dmtree:write(merge_nodes(FoundKey, PKeys, LeftNode, RightNode), Tree), dmtree:delete(RightNode#node.offset, Tree), - % ?infoFmt("new child: ~p~n", [NC]), - % ?infoFmt("new parent: ~p~n", [NP]), + ?infoFmt("new child: ~p~n", [NC]), + ?infoFmt("new parent: ~p~n", [NP]), {merge, NP}; delete_merge(last, _, _, _, _, Right, Tree) -> - % ?infoMsg("not merging~n"), + ?infoMsg("not merging~n"), {nothing, Right}; delete_merge(_, _, _, _, Left, _, Tree) -> %merged leaf is too large, do not merge - % ?infoMsg("not merging~n"), + ?infoMsg("not merging~n"), {nothing, Left}. merge_nodes(FoundKey, PKeys, LeftNode = #node{m=LeftM,keys=LeftKeys,children=LeftChildren}, RightNode = #node{m=RightM,keys=RightKeys,children=RightChildren}) -> elibs/dmerkle.erl @@ -24,7 +24,7 @@ -include("common.hrl"). -include_lib("kernel/include/file.hrl"). --record(dmtree, {file, size=0, virtsize=0, d, blocksize, headersize=0, filename, ops=[], opdict=dict:new(), freepointer=0, rootpointer=0, kfpointers=[]}). +-record(dmtree, {file, size=0, virtsize=0, d, blocksize, headersize=0, filename, ops=[], opdict=dict:new(), freepointer=0, rootpointer=0, kfpointers=[], bigkfpointer=0}). -ifdef(TEST). -include("etest/dmtree_test.erl"). @@ -143,8 +143,8 @@ handle_call(tx_rollback, _From, State = #dmtree{}) -> handle_call(d, _From, State = #dmtree{d=D}) -> {reply, D, State}; -handle_call(root, _From, State = #dmtree{rootpointer=Ptr}) -> - {reply, int_read(Ptr, State), State}; +handle_call(root, _From, State = #dmtree{rootpointer=Ptr,blocksize=BlockSize}) -> + {reply, int_read(Ptr, BlockSize, State), State}; handle_call(block_size, _From, State = #dmtree{blocksize=BlockSize}) -> {reply, BlockSize, State}; @@ -152,15 +152,15 @@ handle_call(block_size, _From, State = #dmtree{blocksize=BlockSize}) -> handle_call({update_root, Node}, _From, State = #dmtree{}) -> {reply, self(), write_header(State#dmtree{rootpointer=offset(Node)})}; -handle_call({write, Node}, _From, State = #dmtree{}) -> - {N, T} = int_write(Node, State), +handle_call({write, Node}, _From, State = #dmtree{blocksize=BlockSize}) -> + {N, T} = int_write(Node, BlockSize, State), {reply, N, T}; handle_call({delete, Offset}, _From, State = #dmtree{}) -> {reply, self(), int_delete(Offset, State)}; -handle_call({read, Offset}, _From, State = #dmtree{}) -> - {reply, int_read(Offset, State), State}; +handle_call({read, Offset}, _From, State = #dmtree{blocksize=BlockSize}) -> + {reply, int_read(Offset, BlockSize, State), State}; handle_call({read_key, Offset}, _From, State = #dmtree{}) -> {reply, int_read_key(Offset, State), State}; @@ -237,21 +237,26 @@ read_header(File) -> {error, Msg} -> {error, Msg} end. -int_read(0, Tree) -> +int_read(0, _, Tree) -> {error, "tried to read a node from the null pointer"}; -int_read(Offset, #dmtree{file=File,d=D,blocksize=BlockSize}) -> +int_read(Offset, BlockSize, #dmtree{file=File,d=D}) -> case file:pread(File, Offset, BlockSize) of {ok, Bin} -> deserialize(Bin, D, Offset); eof -> error_logger:info_msg("hit an eof for offset ~p", [Offset]), undefined; {error, Reason} -> - error_logger:info_msg("error ~p at offset", [Reason, Offset]), + error_logger:info_msg("error ~p at offset ~p", [Reason, Offset]), undefined end. -int_write(Node, Tree = #dmtree{file=File,blocksize=BlockSize}) -> +int_write(Node = #free{}, BlockSize, Tree = #dmtree{size=Size,file=File}) -> + Offset = offset(Node), + {_, Tree2} = add_operation(Offset, serialize(Node, BlockSize), Tree), + {Node, Tree2}; + +int_write(Node, BlockSize, Tree = #dmtree{file=File}) -> % ?infoFmt("int_write ~p~n", [Node]), Offset = offset(Node), Bin = serialize(Node, BlockSize), @@ -260,49 +265,131 @@ int_write(Node, Tree = #dmtree{file=File,blocksize=BlockSize}) -> {offset(Node, Offset3), Tree3}. int_delete(Offset, Tree = #dmtree{file=File,blocksize=BlockSize,freepointer=Pointer}) -> - {_, Tree2} = int_write(#free{offset=Offset,pointer=Pointer}, Tree), + {_, Tree2} = int_write(#free{offset=Offset,pointer=Pointer}, BlockSize, Tree), write_header(Tree2#dmtree{freepointer=Offset}). %this gotta change at some point -int_read_key(Offset, #dmtree{file=File}) -> - file:position(File, Offset), - int_read_key(File, []); - -int_read_key(Keys, [0 | Key]) -> - K = lists:reverse(Key), - % ?infoFmt("int_read_key ~p~n", [K]), - K; - -int_read_key(Keys, Key) -> - case file:read(Keys, 1) of - {ok, <<Char:8>>} -> int_read_key(Keys, [Char|Key]); - Other -> Other +int_read_key(Offset, Tree = #dmtree{file=File,blocksize=BlockSize}) -> + case file:pread(File, Offset, BlockSize) of + {ok, Bin} -> + case lib_misc:zero_split(Bin) of + {Key, _} -> binary_to_list(Key); + _ -> binary_to_list(Bin) ++ int_read_key(Offset+BlockSize, Tree) + end; + eof -> eof; + {error, Reason} -> {error, Reason} end. -% int_write_key(eof, Key, Tree = #dmtree{blocksize=BlockSize}) when length(Key) < BlockSize -> -% +int_write_key(eof, Key, Tree = #dmtree{size=Size,blocksize=BlockSize,kfpointers=Pointers}) when length(Key) < BlockSize -> + N = ?pointer_for_size(length(Key)+1, BlockSize), + % ?infoFmt("int_write_key ~p~nN ~p~nsize ~p~n", [Key, N, ?size_for_pointer(N)]), + case find_free_keypointer(?size_for_pointer(N), N, lists:nthtail(N-1, Pointers), Tree) of + not_found -> + {Ptr, Tree2} = split_freespace(?size_for_pointer(N), BlockSize, Size, Tree), + % ?infoFmt("writing key to ~p~n tree~p~n", [Ptr, Tree2]), + add_operation(Ptr, [Key,0], Tree2); + {Ptr, Tree2} -> add_operation(Ptr, [Key,0], Tree2) + end; int_write_key(Offset, Key, Tree) -> add_operation(Offset, [Key,0], Tree). -int_delete_key(Offset, Key, Tree = #dmtree{}) -> - - Tree. +int_delete_key(Offset, Key, Tree = #dmtree{blocksize=BlockSize,kfpointers=Pointers}) when length(Key) < BlockSize -> + FirstN = ?pointer_for_size(length(Key)+1, BlockSize), + % ?debugHere, + {N, Tree2} = merge_freespace(FirstN, ?size_for_pointer(FirstN), Offset, Tree), + % ?debugHere, + {_, Tree3} = int_write(#free{pointer=lists:nth(N, Pointers),offset=Offset}, ?size_for_pointer(N), Tree2), + % ?debugHere, + write_header(replace_kfpointer(N, Offset, Tree3)). %%-------------------------------- %% ============SUPPORT FUNCTIONS %%-------------------------------- flush(File, Ops) -> file:pwrite(File, lists:reverse(Ops)). + +find_free_keypointer(Size, N, [], Tree) -> + % ?infoMsg("find_free_keypointer not found ~n"), + not_found; + +find_free_keypointer(Size, N, [0|Pointers], Tree) -> + % ?infoFmt("find_free_keypointer Size ~p N ~p [0 | Pointers]~n", [Size, N]), + find_free_keypointer(Size, N+1, Pointers, Tree); + +find_free_keypointer(Size, N, [Ptr|Pointers], Tree) -> + % ?infoFmt("find_free_keypointer found freespace Size ~p N ~p Ptr ~p~n", [Size, N, Ptr]), + Free = int_read(Ptr, Size, Tree), + % ?infoFmt("Free was ~p~n", Free), + {Ptr, Tree2} = split_freespace(Size, ?size_for_pointer(N), Ptr, Tree), + {Ptr, write_header(replace_kfpointer(N, Free#free.pointer, Tree2))}. + +replace_kfpointer(N, Ptr, Tree = #dmtree{kfpointers=Pointers}) -> + Tree#dmtree{kfpointers=lib_misc:nthreplace(N, Ptr, Pointers)}. + +merge_freespace(N, BlockSize, Offset, Tree = #dmtree{blocksize=BlockSize}) -> + % ?infoMsg("merge_freespace got whole block~n"), + {N, Tree}; + +merge_freespace(N, Size, Offset, Tree = #dmtree{file=File,blocksize=BlockSize,headersize=HeaderSize}) when Size < BlockSize, ?block(Offset, HeaderSize, BlockSize) == ?block((Offset+Size), HeaderSize, BlockSize) -> + % ?infoFmt("merge_freespace N ~p Size ~p Offset ~p~n", [N, Size, Offset]), + case file:pread(File, Offset+Size, Size) of + {ok, <<3:8, _/binary>>} -> + Tree2 = remove_free_pointer(Offset+Size, N, Tree), + merge_freespace(N+1, Size * 2, Offset, Tree2); + {ok, Bin} -> {N, Tree}; + eof -> {N, Tree} + end; + +merge_freespace(N, _, _, Tree) -> {N, Tree}. + +remove_free_pointer(Ptr, N, Tree = #dmtree{file=File,kfpointers=Pointers}) -> + Head = lists:nth(N, Pointers), + Size = ?size_for_pointer(N), + Free = int_read(Head, Size, Tree), + ToReplace = int_read(Ptr, Size, Tree), + if + Head == Ptr -> write_header(replace_kfpointer(N, ToReplace#free.pointer, Tree)); + true -> remove_free_pointer(Free, ToReplace, Size, Tree) + end. + +remove_free_pointer(Free = #free{pointer=Ptr}, ToReplace = #free{offset=Ptr,pointer=Next}, Size, Tree = #dmtree{file=File}) -> + {_, Tree2} = int_write(Free#free{pointer=Next}, Size, Tree), + Tree2; + +remove_free_pointer(Free = #free{pointer=0}, _, _, Tree) -> Tree; + +remove_free_pointer(Free = #free{pointer=Ptr}, ToReplace, Size, Tree = #dmtree{file=File}) -> + remove_free_pointer(int_read(Ptr, Size, Tree), ToReplace, Size, Tree). + +split_freespace(ReqSize, ReqSize, Ptr, Tree) -> + {Ptr, Tree}; + +%this is ok because writing past the end fills with zeroes +split_freespace(ReqSize, FreeSize, Ptr, Tree = #dmtree{file=File,blocksize=BlockSize,kfpointers=Pointers}) when FreeSize > ReqSize -> + % ?infoFmt("split_freespace reqsize ~p freesize ~p ptr ~p~n", [ReqSize, FreeSize, Ptr]), + N = ?pointer_for_size(FreeSize div 2, BlockSize), + NxtPtr = lists:nth(N, Pointers), + % ?infoFmt("split_freespace writing free cell of size ~p to ~p~n", [FreeSize div 2, Ptr+(FreeSize div 2)]), + {FreeNode, Tree2} = int_write(#free{pointer=NxtPtr,offset=Ptr+(FreeSize div 2)}, FreeSize div 2, Tree), + % ?infoFmt("file size after free write ~p~n", [Tree2#dmtree.size]), + FreePtr = offset(FreeNode), + Tree3 = write_header(replace_kfpointer(N, FreePtr, Tree2)), + split_freespace(ReqSize, FreeSize div 2, Ptr, Tree3). add_operation(eof, Bin, Tree = #dmtree{file=File,size=Size}) -> BinSize = iolist_size(Bin), ok = file:pwrite(File, Size, Bin), {Size, Tree#dmtree{size=Size+BinSize}}; -add_operation(Offset, Bin, Tree = #dmtree{file=File}) -> +add_operation(Offset, Bin, Tree = #dmtree{file=File,size=Size}) -> + BinSize = iolist_size(Bin), ok = file:pwrite(File, Offset, Bin), - {Offset, Tree}. + if + Offset == Size -> {Offset, Tree#dmtree{size=Size+iolist_size(Bin)}}; + Offset > Size -> {Offset, Tree#dmtree{size=Offset+iolist_size(Bin)}}; + true -> {Offset, Tree} + end. % add_operation(eof, Bin, Tree = #dmtree{size=Size,virtsize=VirtSize,ops=Ops,opdict=Dict}) -> % BinSize = true_size(Bin), @@ -321,7 +408,7 @@ true_size(Bin) when is_binary(Bin) -> true_size(_) -> 1. create_or_read_root(Tree = #dmtree{file=File,blocksize=BlockSize,headersize=HeaderSize,rootpointer=0}) -> - {Root, Tree2} = int_write(#leaf{offset=HeaderSize}, Tree), + {Root, Tree2} = int_write(#leaf{offset=HeaderSize}, BlockSize, Tree), % ?infoFmt("wrote root ~p~n", [Root]), write_header(Tree2#dmtree{rootpointer=offset(Root)}); @@ -330,27 +417,27 @@ create_or_read_root(Tree = #dmtree{file=File,blocksize=BlockSize,rootpointer=Ptr take_free_offset(eof, Tree = #dmtree{file=File,blocksize=BlockSize,freepointer=FreePtr}) when FreePtr > 0 -> Offset = FreePtr, - #free{pointer=NewFreePointer} = int_read(FreePtr, Tree), + #free{pointer=NewFreePointer} = int_read(FreePtr, BlockSize, Tree), {Offset, write_header(Tree#dmtree{freepointer=NewFreePointer})}; take_free_offset(Offset, Tree) -> {Offset, Tree}. -serialize_header(#dmtree{blocksize=BlockSize, freepointer=FreePtr, rootpointer=RootPtr, kfpointers=Pointers}) -> - Preamble = <<?VERSION:8, BlockSize:32, FreePtr:64, RootPtr:64>>, +serialize_header(#dmtree{blocksize=BlockSize, freepointer=FreePtr, bigkfpointer=BKFPointer, rootpointer=RootPtr, kfpointers=Pointers}) -> + Preamble = <<?VERSION:8, BlockSize:32, FreePtr:64, RootPtr:64, BKFPointer:64>>, FreeSpace = (?STATIC_HEADER - byte_size(Preamble))*8, PtrBin = << <<Ptr:64>> || Ptr <- Pointers >>, % ?infoFmt("pointers: ~p~nptrbin~p~nfreespace~p~n", [Pointers, PtrBin, FreeSpace]), <<Preamble/binary, PtrBin/binary, 0:FreeSpace>>. %this will try and match the current version, if it doesn't then we gotta punch out -deserialize_header(<<?VERSION:8, BlockSize:32, FreePtr:64, RootPtr:64, Rest/binary>>) -> +deserialize_header(<<?VERSION:8, BlockSize:32, FreePtr:64, RootPtr:64, BKFPointer:64, Rest/binary>>) -> PointerSize = ?pointers_from_blocksize(BlockSize), <<PBin:PointerSize/binary, _/binary>> = Rest, Pointers = [Ptr || <<Ptr:64>> <= PBin], HeaderSize = ?headersize_from_blocksize(BlockSize), D = ?d_from_blocksize(BlockSize), - {ok, #dmtree{blocksize=BlockSize,d=D,headersize=HeaderSize,freepointer=FreePtr,rootpointer=RootPtr,kfpointers=Pointers}}; + {ok, #dmtree{blocksize=BlockSize,d=D,headersize=HeaderSize,freepointer=FreePtr,rootpointer=RootPtr,kfpointers=Pointers, bigkfpointer=BKFPointer}}; %hit the canopy deserialize_header(BinHeader) -> elibs/dmtree.erl @@ -4,17 +4,31 @@ -define(OFFSET_BASIS, 2166136261). -define(FNV_PRIME, 16777619). --export([pmap/3, hash/1, nthdelete/2, rand_str/1, position/2, shuffle/1, floor/1, ceiling/1, time_to_epoch_int/1, time_to_epoch_float/1, now_int/0, now_float/0, byte_size/1, listify/1, reverse_bits/1]). +-export([pmap/3, hash/1, nthdelete/2, zero_split/1, nthreplace/3, rand_str/1, position/2, shuffle/1, floor/1, ceiling/1, time_to_epoch_int/1, time_to_epoch_float/1, now_int/0, now_float/0, byte_size/1, listify/1, reverse_bits/1]). -ifdef(TEST). -include("etest/lib_misc_test.erl"). -endif. +zero_split(Bin) -> + zero_split(0, Bin). + +zero_split(N, Bin) when N > byte_size(Bin) -> Bin; + +zero_split(N, Bin) -> + case Bin of + <<_:N/binary, 0:8, _/binary>> -> split_binary(Bin, N); + _ -> zero_split(N+1, Bin) + end. + rand_str(N) -> lists:map(fun(I) -> random:uniform(26) + $a - 1 end, lists:seq(1,N)). +nthreplace(N, E, List) -> + lists:sublist(List, N-1) ++ [E] ++ lists:nthtail(N, List). + nthdelete(N, List) -> nthdelete(N, List, []). elibs/lib_misc.erl @@ -205,6 +205,32 @@ live_test_() -> KeyDiff = LeafDiff end}]}. +insert_variable_keys_under_block_size_test() -> + test_cleanup(), + {ok, Pid} = open(data_file(), 256), + Keys = lists:map(fun(N) -> + lists:duplicate(N, $a) + end, lists:seq(1,255)), + lists:foreach(fun(Key) -> + update(Key, "value", Pid) + end, Keys), + {LK, _} = lists:unzip(leaves(Pid)), + ?assertEqual(Keys, lists:sort(LK)), + close(Pid). + +insert_variable_keys_over_a_block_size_test() -> + test_cleanup(), + {ok, Pid} = open(data_file(), 256), + Keys = lists:map(fun(N) -> + lists:duplicate(N, $a) + end, lists:seq(1,512)), + lists:foreach(fun(Key) -> + update(Key, "value", Pid) + end, Keys), + {LK, _} = lists:unzip(leaves(Pid)), + ?assertEqual(Keys, lists:sort(LK)), + close(Pid). + simple_deletion_test() -> test_cleanup(), {ok, Pid} = open(data_file(), 256), @@ -308,6 +334,25 @@ partial_deletion_and_rebuild_test() -> ?assertEqual(IdxSize, filelib:file_size(data_file())), close(Pid). +partial_delete_and_rebuild_var_keysize_small_test() -> + test_cleanup(), + {ok, Pid} = open(data_file(), 256), + Keys = lists:map(fun(N) -> + lists:duplicate(N, $a) + end, lists:seq(1,255)), + lists:foreach(fun(Key) -> + update(Key, "value", Pid) + end, Keys), + Size = filelib:file_size(data_file()), + lists:foreach(fun(Key) -> + delete(Key, Pid) + end, lists:sublist(Keys, 50)), + lists:foreach(fun(Key) -> + update(Key, "Value", Pid) + end, lists:sublist(Keys, 50)), + ?assertEqual(Size, filelib:file_size(data_file())), + close(Pid). + open_and_insert_n(N) -> test_cleanup(), {ok, Pid} = open(data_file(), 256), etest/dmerkle_test.erl @@ -57,3 +57,12 @@ leaf_round_trip_test() -> offset=0 }, Leaf = deserialize(serialize(Leaf, 81), 4, 0). + +pointers_for_blocksize_test() -> + ?assertEqual(5, ?pointers_from_blocksize(256)), + ?assertEqual(1, ?pointers_from_blocksize(16)). + +size_for_pointer_test() -> + ?assertEqual(16, ?size_for_pointer(1)), + ?assertEqual(256, ?size_for_pointer(5)). + etest/dmtree_test.erl @@ -22,4 +22,10 @@ nthdelete_test() -> ?assertEqual([1,2,3,4,5], nthdelete(0, A)), ?assertEqual([1,2,3,4,5], nthdelete(6, A)), ?assertEqual([2,3,4,5], nthdelete(1, A)), - ?assertEqual([1,2,4,5], nthdelete(3, A)). \ No newline at end of file + ?assertEqual([1,2,4,5], nthdelete(3, A)). + +zero_split_test() -> + ?assertEqual({<<"">>, <<0,"abcdefg">>}, zero_split(<<0, "abcdefg">>)), + ?assertEqual({<<"abd">>, <<0, "efg">>}, zero_split(<<"abd", 0, "efg">>)), + ?assertEqual({<<"abcdefg">>, <<0>>}, zero_split(<<"abcdefg",0>>)), + ?assertEqual(<<"abcdefg">>, zero_split(<<"abcdefg">>)). \ No newline at end of file etest/lib_misc_test.erl @@ -1,9 +1,13 @@ -define(VERSION, 1). --define(STATIC_HEADER, 85). +-define(STATIC_HEADER, 93). -define(d_from_blocksize(BlockSize), trunc((BlockSize - 17)/16)). --define(pointers_from_blocksize(BlockSize), (trunc(math:sqrt(BlockSize)) - 4)). +-define(pointers_from_blocksize(BlockSize), (lib_misc:ceiling(math:log(BlockSize)/math:log(2)) - 3)). +-define(pointer_for_size(Size, BlockSize), (if Size =< 16 -> 1; Size =< BlockSize -> ?pointers_from_blocksize(Size); true -> last end)). +-define(size_for_pointer(N), (2 bsl (N+2))). -define(headersize_from_blocksize(BlockSize), (?STATIC_HEADER + ?pointers_from_blocksize(BlockSize) * 8)). +-define(aligned(Ptr, HeaderSize, BlockSize), (((Ptr - (HeaderSize)) rem BlockSize) == 0)). +-define(block(Ptr, HeaderSize, BlockSize), ((Ptr - (HeaderSize)) div BlockSize)). -record(node, {m=0, keys=[], children=[], offset=eof}). -record(leaf, {m=0, values=[], offset=eof}). include/dmerkle.hrl 72b77c3c81c55de4796acc5b8ca5e28343dfe61e Cliff Moon cliff@moonpolysoft.com http://github.com/cliffmoon/dynomite/commit/2ab74d4560ec89bda3d3a40a7ee1ea1f62d8f32f 2ab74d4560ec89bda3d3a40a7ee1ea1f62d8f32f 2009-01-26T21:19:35-08:00 2009-01-26T21:19:35-08:00 refactoring complete for now f38b69e79453db3de4e0c3703982c4dc984d98c2 Cliff Moon cliff@moonpolysoft.com