Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
tree: d617d0ec70
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

file 130 lines (106 sloc) 3.708 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
%% Copyright (c) 2006-2009 Joe Armstrong
%% See MIT-LICENSE for licensing information.

-module(elib1_fast_read).
-compile(export_all).
-export([foldl/3, open_abs/1, close_abs/1, read_abs/2]).

%% foldl(File, Fun, Acc) -> Acc1
%% calls Fun(Tup, Pos, Acc) -> Acc1
%% for each tuple in <File>
%% File is a list of tuples encoded with
%% <<Len:32/integer, term_to_binary(Tup)>>
%% Pos is the *absolute* position in the file
%% of the 4 byte header
%% open_abs(File) -> Stream
%% close_abs(Stream) -> void
%% read_abs(Stream, Pos) -> Term

%%----------------------------------------------------------------------
%% @doc Fold a fun over a file of terms.
%% The terms are stored in binary term format.
%% Fun is of the form <b>F(Term, Pos, Acc) -> NewAcc</b>
%% Terms are read one at a time from the file. Pos is an integer
%% index into the file this can be used later as an argument
%% to <b>read_abs</b>

-type acc() :: any().

-type filename() :: string().

-type stream()::file:io_device().

%% Acc is an accumulator

-spec foldl(filename(),
fun((any(), Pos::integer(), acc()) -> acc()),
acc()) -> acc().

foldl(File, Fun, Acc0) ->
    %% io:format("opening:~p~n",[File]),
    case file:open(File, [read,raw,binary,{read_ahead, 64000}]) of
{ok, Stream} ->
try read2a(Stream, Fun, Acc0, 0, <<>>) of
Acc1 ->
file:close(Stream),
Acc1
catch
throw:X ->
file:close(Stream),
X
end;
_ ->
%% couldn't open the file so ...
io:format("warning could not open ~p~n",[File]),
Acc0
    end.

%% there is one unit test for elib1_fast_read and elib_fast_write

foldl_test() ->
    Stream = elib1_fast_write:new("./tmp.tmp"),
    S1 = elib1_fast_write:write(Stream, {term,1}),
    S2 = elib1_fast_write:write(S1, "it works"),
    S3 = elib1_fast_write:write(S2, {term,2,more,stuff}),
    elib1_fast_write:close(S3),
    %% Now recover all the elements in the stream
    L = foldl("./tmp.tmp", fun(Term, Pos, A) -> [{Pos, Term}|A] end, []),
    [{Pos3,{term,2,more,stuff}},{Pos2,"it works"},{Pos1,{term,1}}] = L,
    Stream1 = open_abs("./tmp.tmp"),
    "it works" = read_abs(Stream1, Pos2),
    {term,1} = read_abs(Stream1, Pos1),
    {term,2,more,stuff} = read_abs(Stream1, Pos3),
    close_abs(Stream1),
    ok.

%%----------------------------------------------------------------------
%% @doc Create a stream from a file.

-spec open_abs(filename()) -> stream().
    
open_abs(File) ->
    {ok, Stream} = file:open(File, [read,raw,binary,{read_ahead, 64000}]),
    Stream.

%%----------------------------------------------------------------------
%% @doc Close a stream.

-spec close_abs(stream()) -> ok.
    
close_abs(Stream) ->
    file:close(Stream).

%%----------------------------------------------------------------------
%% @doc Read a term given the absolute position from a stream.

-spec read_abs(stream(), Pos::integer()) -> any().

read_abs(Stream, Pos) ->
    {ok, <<I:32/big>>} = file:pread(Stream, Pos, 4),
    {ok, Bin} = file:pread(Stream, Pos+4, I),
    binary_to_term(Bin).

read2a(Stream, Fun, A0, Pos, <<Size:32/big,B/binary>>=Bx) ->
    case size(B) of
N when N >= Size ->
{Ba, Bb} = split_binary(B, Size),
T = binary_to_term(Ba),
%% This is where we call the user's function
A1 = Fun(T, Pos, A0),
read2a(Stream, Fun, A1, Pos + Size + 4, Bb);
_ ->
case
file:read(Stream, 64000) of
{ok, Bc} ->
read2a(Stream, Fun, A0, Pos, <<Bx/binary, Bc/binary>>);
eof ->
A0
end
    end;
read2a(Stream, Fun, A0, Pos, Bx) ->
        case
file:read(Stream, 64000) of
{ok, Bc} ->
read2a(Stream, Fun, A0, Pos, <<Bx/binary, Bc/binary>>);
eof ->
A0
end.


Something went wrong with that request. Please try again.