Browse files

ref deltas now properly unpacked, too

  • Loading branch information...
1 parent 8919d83 commit 0546f995f946aae6085f27219f58ecfe36ac2031 @schacon committed Dec 3, 2009
Showing with 62 additions and 46 deletions.
  1. +1 −1 src/git.erl
  2. +49 −45 src/packfile.erl
  3. +12 −0 t/005-cat-file-packed_ref_delta.t
View
2 src/git.erl
@@ -80,7 +80,7 @@ extract_loose_object_data(CompData) ->
get_packfile_object_data(Git, ObjectSha) ->
case find_packfile_with_object(Git, ObjectSha) of
{ok, PackFilePath, Offset} ->
- packfile:get_packfile_data(PackFilePath, Offset);
+ packfile:get_packfile_data(Git, PackFilePath, Offset);
_Else ->
invalid
end.
View
94 src/packfile.erl
@@ -3,40 +3,42 @@
%%
-module(packfile).
--export([get_packfile_data/2]).
+-export([get_packfile_data/3]).
-get_packfile_data(PackFilePath, Offset) ->
+get_packfile_data(Git, PackFilePath, Offset) ->
case file:open(PackFilePath, [read, binary]) of
{ok, IoDevice} ->
- read_packfile_object_offset(IoDevice, Offset);
+ read_packfile_object_offset(Git, IoDevice, Offset);
{error, _Reason} ->
error
end.
-read_packfile_object_offset(IoDevice, Offset) ->
+read_packfile_object_offset(Git, IoDevice, Offset) ->
file:position(IoDevice, Offset),
{ok, Byte} = file:read(IoDevice, 1),
<<ContinueBit:1, Type:3, InitSize:4>> = Byte,
TypeTerm = type_int_to_term(Type),
%io:fwrite("Object Data: ~p ~p ~p~n", [ContinueBit, Type, InitSize]),
- {FinalType, Size, Data} = read_object_data(IoDevice, ContinueBit, InitSize, 4, Offset, Offset + 1, TypeTerm),
- io:fwrite("Final Object Data: ~p ~p ~p~n", [Size, FinalType, Data]),
+ {FinalType, Size, Data} = read_object_data(Git, IoDevice, ContinueBit, InitSize, 4, Offset, Offset + 1, TypeTerm),
+ %io:fwrite("Final Object Data: ~p ~p ~p~n", [Size, FinalType, Data]),
{FinalType, Size, Data}.
-read_object_data(IoDevice, 1, Size, Offset, OrigOffset, FileOffset, TypeTerm) ->
+read_object_data(Git, IoDevice, 1, Size, Shift, OrigOffset, FileOffset, TypeTerm) ->
{ok, Byte} = file:read(IoDevice, 1),
<<ContinueBit:1, NextSize:7>> = Byte,
% bit shift the size
- SizeOr = NextSize bsl Offset,
+ SizeOr = NextSize bsl Shift,
NewSize = Size bor SizeOr,
- NextOffset = Offset + 7,
- %io:fwrite("Object Data: ~p ~p (~p)~n", [ContinueBit, NextSize, NewSize]),
- read_object_data(IoDevice, ContinueBit, NewSize, NextOffset, OrigOffset, FileOffset + 1, TypeTerm);
-read_object_data(IoDevice, 0, Size, Offset, OrigOffset, FileOffset, ofs_delta) ->
- read_ofs_deltified_object_data(IoDevice, Size, Offset, FileOffset, OrigOffset);
-read_object_data(IoDevice, 0, Size, Offset, OrigOffset, FileOffset, ref_delta) ->
- read_ref_deltified_object_data(IoDevice, Size, Offset, FileOffset, OrigOffset);
-read_object_data(IoDevice, 0, Size, _Offset, _OrigOffset, _FileOffset, NormalType) ->
+ NextShift = Shift + 7,
+ %io:fwrite("Object Data: [~p] ~p ~p (~p)~n", [FileOffset, ContinueBit, NextSize, NewSize]),
+ read_object_data(Git, IoDevice, ContinueBit, NewSize, NextShift, OrigOffset, FileOffset + 1, TypeTerm);
+read_object_data(Git, IoDevice, 0, Size, Shift, OrigOffset, FileOffset, ofs_delta) ->
+ read_ofs_deltified_object_data(Git, IoDevice, Size, Shift, FileOffset, OrigOffset);
+read_object_data(Git, IoDevice, 0, Size, Shift, OrigOffset, FileOffset, ref_delta) ->
+ read_ref_deltified_object_data(Git, IoDevice, Size, Shift, FileOffset, OrigOffset);
+read_object_data(_Git, IoDevice, 0, Size, _Shift, _OrigOffset, FileOffset, NormalType) ->
+ %io:fwrite("File Offset:~p~n", [FileOffset]),
+ file:position(IoDevice, FileOffset),
Z = zlib:open(),
ok = zlib:inflateInit(Z),
Data = inflate_object_data(Z, IoDevice, []),
@@ -45,13 +47,25 @@ read_object_data(IoDevice, 0, Size, _Offset, _OrigOffset, _FileOffset, NormalTyp
zlib:close(Z),
{NormalType, Size, Data}.
-read_ofs_deltified_object_data(IoDevice, Size, Offset, FileOffset, OrigOffset) ->
+% git rev-list --objects --all | git pack-objects --window=250 --depth=250 --stdout > #{packfile_pack}
+% git index-pack -o #{packfile_idx} #{packfile_pack}
+read_ref_deltified_object_data(Git, IoDevice, Size, Offset, FileOffset, OrigOffset) ->
+ {ok, Sha} = file:read(IoDevice, 20),
+ HexSha = hex:bin_to_hexstr(Sha),
+ {TypeTerm, BaseSize, BaseData} = git:read_object(Git, HexSha),
+ %io:fwrite("Base:~p~n", [BaseSize]),
+ {TypeTerm, DeltaSize, DeltaData} = read_object_data(Git, IoDevice, 0, Size, Offset, OrigOffset, FileOffset + 20, TypeTerm),
+ %io:fwrite("DeltaData: ~p:~p:~p~n", [TypeTerm, DeltaSize, DeltaData]),
+ PatchedData = patch_delta(DeltaData, BaseData),
+ {TypeTerm, length(PatchedData), list_to_binary(PatchedData)}.
+
+read_ofs_deltified_object_data(Git, IoDevice, Size, Offset, FileOffset, OrigOffset) ->
{ok, BaseOffset, BytesRead} = get_base_offset(IoDevice),
NewOffset = OrigOffset - BaseOffset,
%io:fwrite("BaseOffset: ~p:~p:~p~n", [NewOffset, OrigOffset, BaseOffset]),
- {TypeTerm, BaseSize, BaseData} = read_packfile_object_offset(IoDevice, NewOffset),
+ {TypeTerm, BaseSize, BaseData} = read_packfile_object_offset(Git, IoDevice, NewOffset),
file:position(IoDevice, FileOffset + BytesRead),
- {TypeTerm, _DeltaSize, DeltaData} = read_object_data(IoDevice, 0, Size, Offset, OrigOffset, FileOffset + BytesRead, TypeTerm),
+ {TypeTerm, _DeltaSize, DeltaData} = read_object_data(Git, IoDevice, 0, Size, Offset, OrigOffset, FileOffset + BytesRead, TypeTerm),
%io:fwrite("DeltaData: ~p:~p:~p~n", [TypeTerm, DeltaSize, DeltaData]),
PatchedData = patch_delta(DeltaData, BaseData),
{TypeTerm, length(PatchedData), list_to_binary(PatchedData)}.
@@ -61,43 +75,41 @@ patch_delta(DeltaData, BaseData) ->
BaseList = binary_to_list(BaseData),
{ok, SrcSize, PosA} = patch_delta_header_size(DeltaList, 1),
% SrcSize should == BaseData.size
- io:fwrite("PatchA: ~p:~p~n", [SrcSize, PosA]),
+ %io:fwrite("Patch:SrcSize : ~p:~p~n", [SrcSize, PosA]),
{ok, DestSize, PosB} = patch_delta_header_size(DeltaList, PosA),
- io:fwrite("PatchB: ~p:~p~n", [DestSize, PosB]),
+ %io:fwrite("Patch:DestSize: ~p:~p~n", [DestSize, PosB]),
PatchedData = patch_delta([], BaseList, DeltaList, PosB, length(DeltaList) + 1),
- io:fwrite("PatchData: ~p~n", [length(PatchedData)]),
+ %io:fwrite("PatchData: ~p~n", [length(PatchedData)]),
PatchedData.
patch_delta(PatchedList, _BaseList, _DeltaList, _DeltaPos, _DeltaPos) ->
PatchedList;
patch_delta(PatchedList, BaseList, DeltaList, DeltaPos, Size) ->
- io:fwrite("Patch Delta ~p:~p~n", [DeltaPos, DeltaList]),
Byte = lists:nth(DeltaPos, DeltaList),
- io:fwrite("Patch Delta DataA~n"),
+ StartPos = DeltaPos + 1,
<<PatchBit:1, PatchData:7>> = <<Byte>>,
+ %io:fwrite("Patch: ~p:~p:~p~n", [PatchBit, DeltaPos, Size]),
case PatchBit of
1 ->
% append data from base list
<<_Pb:1, S16:1, S8:1, S0:1, Of24:1, Of16:1, Of8:1, Of0:1>> = <<Byte>>,
- StartPos = DeltaPos + 1,
{CpOffA, DeltaOffA} = calc_cp(DeltaList, StartPos, Of0, 0, 0),
{CpOffB, DeltaOffB} = calc_cp(DeltaList, DeltaOffA, Of8, 8, CpOffA),
{CpOffC, DeltaOffC} = calc_cp(DeltaList, DeltaOffB, Of16, 16, CpOffB),
{CpOff, DeltaOffD} = calc_cp(DeltaList, DeltaOffC, Of24, 24, CpOffC),
{CpSizeA, DeltaOffE} = calc_cp(DeltaList, DeltaOffD, S0, 0, 0),
{CpSizeB, DeltaOffF} = calc_cp(DeltaList, DeltaOffE, S8, 8, CpSizeA),
{CpSize, DeltaOff} = calc_cp(DeltaList, DeltaOffF, S16, 16, CpSizeB),
- io:fwrite("Off/Size: ~p:~p:~p~n", [CpOff, CpSize, DeltaOff]),
+ %io:fwrite("Off/Size: ~p:~p:~p~n", [CpOff, CpSize, DeltaOff]),
PatchDataList = lists:sublist(BaseList, CpOff + 1, CpSize),
patch_delta(PatchedList ++ PatchDataList, BaseList, DeltaList, DeltaOff, Size);
0 ->
% append data from delta list
- io:fwrite("Patch Delta Data~n"),
- PatchDataList = lists:sublist(DeltaList, DeltaPos, PatchData),
- patch_delta(PatchedList ++ PatchDataList, BaseList, DeltaList, DeltaPos + PatchData, Size)
+ %io:fwrite("Patch Delta Data (~p:~p)~n", [StartPos, PatchData]),
+ PatchDataList = lists:sublist(DeltaList, StartPos, PatchData),
+ patch_delta(PatchedList ++ PatchDataList, BaseList, DeltaList, StartPos + PatchData, Size)
end.
-
calc_cp(DList, DPos, 1, Shift, Cp) ->
Data = lists:nth(DPos, DList),
ShiftCp = Data bsl Shift,
@@ -112,7 +124,7 @@ patch_delta_header_size(DeltaData, Pos) ->
patch_delta_header_size(DeltaData, Pos, Shift, 1, Size) ->
Byte = lists:nth(Pos, DeltaData),
<<ContinueBit:1, NextSize:7>> = <<Byte>>,
- io:fwrite("Byte: ~p:~p:~p:~p:~p:~p~n", [Byte, Pos, Shift, Size, ContinueBit, NextSize]),
+ %io:fwrite("Byte: ~p:~p:~p:~p:~p:~p~n", [Byte, Pos, Shift, Size, ContinueBit, NextSize]),
SizeOr = NextSize bsl Shift,
NewSize = Size bor SizeOr,
patch_delta_header_size(DeltaData, Pos + 1, Shift + 7, ContinueBit, NewSize);
@@ -135,30 +147,22 @@ get_base_offset(IoDevice, 1, Offset, BytesRead) ->
get_base_offset(_IoDevice, 0, Offset, BytesRead) ->
{ok, Offset, BytesRead}.
-
-read_ref_deltified_object_data(IoDevice, _Size, _Offset, _FileOffset, _OrigOffset) ->
- {ok, Sha} = file:read(IoDevice, 20),
- _HexSha = hex:bin_to_hexstr(Sha),
- % find the offset of HexSha, run read_object_data again
- % {Size, Data} = get_packfile_data(Path, Offset),
- % DeltaData = read_object_data(%here),
- % patch_delta(Data, DeltaData)
- {100, <<"Ref Data">>}.
-
inflate_object_data(Z, IoDevice, SoFar) ->
case file:read(IoDevice, 4096) of
{ok, Bytes} ->
- _Inflated = case catch zlib:inflate(Z, Bytes) of
+ Inflated = case catch zlib:inflate(Z, Bytes) of
{'EXIT', {'data_error', _Backtrace} } ->
- io:format("zlib:inflate data_error,~n"),
+ %io:format("zlib:inflate data_error,~n"),
SoFar;
{'EXIT', Reason} ->
- io:format("zlib:inflate error -> [~p]~n", [Reason]),
+ %io:format("zlib:inflate error -> [~p]~n", [Reason]),
+ SoFar;
+ [] ->
SoFar;
Iolist ->
[Data] = Iolist,
ListData = list_to_binary([binary_to_list(Data)|SoFar]),
- io:fwrite("Size: ~p~n", [size(ListData)]),
+ %io:fwrite("Size: ~p~n", [size(ListData)]),
inflate_object_data(Z, IoDevice, ListData)
end;
_Else ->
View
12 t/005-cat-file-packed_ref_delta.t
@@ -0,0 +1,12 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -sasl -boot start_sasl -noshell
+
+main(_) ->
+ Git = git:open("test_git"),
+ {Type, Size, Data} = git:read_object(Git, "ff69c3684a18592c741332b290492aa39d980e02"), %% blob
+ %{Type, Size, Data} = git:read_object(Git, "fb9ce46625103fd5c208f2a04f0738cc1f449e4d"), %% blob
+ io:fwrite("Data: ~p ~p~n", [Type, Size]),
+ io:fwrite("Data: ~p~n", [Data]),
+ io:fwrite("Data: ~s~n", [binary_to_list(Data)]).
+

0 comments on commit 0546f99

Please sign in to comment.