Skip to content
Newer
Older
100644 215 lines (192 sloc) 6.95 KB
54a3ecd @dizzyd Updating license header on all erlang code
dizzyd authored Jul 12, 2010
1 %% -------------------------------------------------------------------
2 %%
22a71ea @rustyio Add license information.
rustyio authored Aug 17, 2011
3 %% merge_index: main interface to merge_index library.
4 %%
5 %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved.
6 %%
7 %% This file is provided to you under the Apache License,
8 %% Version 2.0 (the "License"); you may not use this file
9 %% except in compliance with the License. You may obtain
10 %% a copy of the License at
11 %%
12 %% http://www.apache.org/licenses/LICENSE-2.0
13 %%
14 %% Unless required by applicable law or agreed to in writing,
15 %% software distributed under the License is distributed on an
16 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 %% KIND, either express or implied. See the License for the
18 %% specific language governing permissions and limitations
19 %% under the License.
20 %%
21 %% -------------------------------------------------------------------
22
23 %% @doc The merge_index application is an index from Index/Field/Term
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
24 %% (IFT) tuples to Values. The values are document IDs or some form
25 %% of identification for the object which contains the IFT.
26 %% Futhermore, each IFT/Value pair has an associated proplists (Props)
27 %% and timestamp (Timestamp) which are used to describe where the IFT
28 %% was found in the Value and at what time the entry was written,
29 %% respectively.
22a71ea @rustyio Add license information.
rustyio authored Aug 17, 2011
30
32f390d Incorporate merge_index version 2.
Rusty Klophaus authored Apr 7, 2010
31 -module(merge_index).
32 -author("Rusty Klophaus <rusty@basho.com>").
33 -include("merge_index.hrl").
34 -include_lib("kernel/include/file.hrl").
35
36 -export([
37 %% API
f82cc67 Fix merge_index to use app env vars, fix app.config typo.
Rusty Klophaus authored Aug 4, 2010
38 start_link/1,
556b5c5 Update merge_index to meet new search backend requirements.
Rusty Klophaus authored Jul 18, 2010
39 stop/1,
96bb3bb @rzezeski Remove unused API index/7
rzezeski authored May 16, 2011
40 index/2,
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
41 lookup/5, lookup_sync/5,
42 range/7, range_sync/7,
02941ad Remove subtypes and subterms from code.
Rusty Klophaus authored Jun 14, 2010
43 info/4,
2229337 Continue work on merge_index handoff.
Rusty Klophaus authored Apr 28, 2010
44 is_empty/1,
45 fold/3,
33d721b Continue merge_index updates.
Rusty Klophaus authored Jul 19, 2010
46 drop/1,
47 compact/1
32f390d Incorporate merge_index version 2.
Rusty Klophaus authored Apr 7, 2010
48 ]).
49
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
50 -type(index() :: any()).
51 -type(field() :: any()).
52 -type(mi_term() :: any()).
53 -type(size() :: all | integer()).
54 -type(posting() :: {Index::index(),
55 Field::field(),
56 Term::mi_term(),
57 Value::any(),
58 Props::[proplists:property()],
59 Timestamp::integer()}).
4b60747 @rzezeski Add an explicit error return to lookup/range
rzezeski authored May 27, 2011
60 -type(iterator() :: fun(() -> {any(), iterator()}
61 | eof
62 | {error, Reason::any()})).
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
63
64 -define(LOOKUP_TIMEOUT, 60000).
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
65
66 %% @doc Start a new merge_index server.
67 -spec start_link(string()) -> {ok, Pid::pid()} | ignore | {error, Error::any()}.
4f8a200 @rzezeski General cleanup - add API to `mi_server`
rzezeski authored Aug 19, 2011
68 start_link(Root) -> mi_server:start_link(Root).
32f390d Incorporate merge_index version 2.
Rusty Klophaus authored Apr 7, 2010
69
e88b812 @rzezeski Implement stop
rzezeski authored May 11, 2011
70 %% @doc Stop the merge_index server.
71 -spec stop(pid()) -> ok.
4f8a200 @rzezeski General cleanup - add API to `mi_server`
rzezeski authored Aug 19, 2011
72 stop(Server) -> mi_server:stop(Server).
556b5c5 Update merge_index to meet new search backend requirements.
Rusty Klophaus authored Jul 18, 2010
73
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
74 %% @doc Index `Postings'.
75 -spec index(pid(), [posting()]) -> ok.
4f8a200 @rzezeski General cleanup - add API to `mi_server`
rzezeski authored Aug 19, 2011
76 index(Server, Postings) -> mi_server:index(Server, Postings).
32f390d Incorporate merge_index version 2.
Rusty Klophaus authored Apr 7, 2010
77
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
78 %% @doc Return a `Weight' for the given IFT.
79 -spec info(pid(), index(), field(), mi_term()) ->
80 {ok, [{Term::any(), Weight::integer()}]}.
4f8a200 @rzezeski General cleanup - add API to `mi_server`
rzezeski authored Aug 19, 2011
81 info(Server, Index, Field, Term) -> mi_server:info(Server, Index, Field, Term).
32f390d Incorporate merge_index version 2.
Rusty Klophaus authored Apr 7, 2010
82
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
83 %% @doc Lookup the results for IFT and return an iterator. This
84 %% allows the caller to process data as it comes in/wants it.
85 %%
86 %% @throws lookup_timeout
87 %%
88 %% `Server' - Pid of the server instance.
89 %%
90 %% `Filter' - Function used to filter the results.
91 -spec lookup(pid(), index(), field(), mi_term(), function()) -> iterator().
92 lookup(Server, Index, Field, Term, Filter) ->
93 {ok, Ref} = mi_server:lookup(Server, Index, Field, Term, Filter),
94 make_result_iterator(Ref).
95
96 %% @doc Lookup the results for IFT and return a list. The caller will
97 %% block until the result list is built.
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
98 %%
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
99 %% @throws lookup_timeout
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
100 %%
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
101 %% `Server' - Pid of the server instance.
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
102 %%
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
103 %% `Filter' - Function used to filter the results.
4b60747 @rzezeski Add an explicit error return to lookup/range
rzezeski authored May 27, 2011
104 -spec lookup_sync(pid(), index(), field(), mi_term(), function()) ->
105 list() | {error, Reason::any()}.
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
106 lookup_sync(Server, Index, Field, Term, Filter) ->
107 {ok, Ref} = mi_server:lookup(Server, Index, Field, Term, Filter),
108 make_result_list(Ref).
1c50f94 First cut at merge_index handoff.
Rusty Klophaus authored Apr 28, 2010
109
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
110 %% @doc Much like `lookup' except allows one to specify a range of
111 %% terms. The range is a closed interval meaning that both
112 %% `StartTerm' and `EndTerm' are included.
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
113 %%
114 %% `StartTerm' - The start of the range.
115 %%
116 %% `EndTerm' - The end of the range.
117 %%
118 %% `Size' - The size of the term in bytes.
119 %%
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
120 %% @see lookup/5.
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
121 -spec range(pid(), index(), field(), mi_term(), mi_term(),
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
122 size(), function()) -> iterator().
123 range(Server, Index, Field, StartTerm, EndTerm, Size, Filter) ->
124 {ok, Ref} = mi_server:range(Server, Index, Field, StartTerm, EndTerm,
125 Size, Filter),
126 make_result_iterator(Ref).
127
128 %% @doc Much like `lookup_sync' except allows one to specify a range
129 %% of terms. The range is a closed interval meaning that both
130 %% `StartTerm' and `EndTerm' are included.
131 %%
132 %% `StartTerm' - The start of the range.
133 %%
134 %% `EndTerm' - The end of the range.
135 %%
136 %% `Size' - The size of the term in bytes.
137 %%
138 %% @see lookup_sync/5.
139 -spec range_sync(pid(), index(), field(), mi_term(), mi_term(),
4b60747 @rzezeski Add an explicit error return to lookup/range
rzezeski authored May 27, 2011
140 size(), function()) -> list() | {error, Reason::any()}.
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
141 range_sync(Server, Index, Field, StartTerm, EndTerm, Size, Filter) ->
142 {ok, Ref} = mi_server:range(Server, Index, Field, StartTerm, EndTerm,
143 Size, Filter),
144 make_result_list(Ref).
12f4519 Move condor code back to merge_index, clean up Makefile hacks.
Rusty Klophaus authored Sep 3, 2010
145
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
146 %% @doc Predicate to determine if the buffers AND segments are empty.
147 -spec is_empty(pid()) -> boolean().
4f8a200 @rzezeski General cleanup - add API to `mi_server`
rzezeski authored Aug 19, 2011
148 is_empty(Server) -> mi_server:is_empty(Server).
2229337 Continue work on merge_index handoff.
Rusty Klophaus authored Apr 28, 2010
149
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
150 %% @doc Fold over all IFTs in the index.
151 %%
b5ba9a0 @rzezeski Fix edoc for fold
rzezeski authored May 11, 2011
152 %% `Fun' - Function to fold over data. It takes 7 args. 1-6 are `I',
153 %% `F', `T', `Value', `Props', `Timestamp' and the 7th is the
154 %% accumulator.
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
155 %%
156 %% `Acc' - The accumulator to seed the fold with.
157 %%
158 %% `Acc2' - The final accumulator.
159 -spec fold(pid(), function(), any()) -> {ok, Acc2::any()}.
4f8a200 @rzezeski General cleanup - add API to `mi_server`
rzezeski authored Aug 19, 2011
160 fold(Server, Fun, Acc) -> mi_server:fold(Server, Fun, Acc).
2229337 Continue work on merge_index handoff.
Rusty Klophaus authored Apr 28, 2010
161
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
162 %% @doc Drop all current state and start from scratch.
163 -spec drop(pid()) -> ok.
4f8a200 @rzezeski General cleanup - add API to `mi_server`
rzezeski authored Aug 19, 2011
164 drop(Server) -> mi_server:drop(Server).
33d721b Continue merge_index updates.
Rusty Klophaus authored Jul 19, 2010
165
a0c9aa5 @rzezeski Add some edoc/spec love around the API
rzezeski authored May 9, 2011
166 %% @doc Perform compaction of segments if needed.
167 %%
168 %% `Segs' - The number of segments compacted.
169 %%
170 %% `Bytes' - The number of bytes compacted.
171 -spec compact(pid()) ->
172 {ok, Segs::integer(), Bytes::integer()}
173 | {error, Reason::any()}.
4f8a200 @rzezeski General cleanup - add API to `mi_server`
rzezeski authored Aug 19, 2011
174 compact(Server) -> mi_server:compact(Server).
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
175
176 %%%===================================================================
177 %%% Internal Functions
178 %%%===================================================================
179
180 %% @private
181 make_result_iterator(Ref) ->
182 fun() -> result_iterator(Ref) end.
183
184 %% @private
185 result_iterator(Ref) ->
186 receive
187 {results, Results, Ref} ->
188 {Results, fun() -> result_iterator(Ref) end};
189 {eof, Ref} ->
4b60747 @rzezeski Add an explicit error return to lookup/range
rzezeski authored May 27, 2011
190 eof;
191 {error, Ref, Reason} ->
192 {error, Reason}
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
193 after
194 ?LOOKUP_TIMEOUT ->
195 throw(lookup_timeout)
196 end.
197
198 %% @private
199 make_result_list(Ref) ->
200 make_result_list(Ref, []).
201
202 %% @private
203 make_result_list(Ref, Acc) ->
204 receive
205 {results, Results, Ref} ->
206 make_result_list(Ref, [Results|Acc]);
207 {eof, Ref} ->
4b60747 @rzezeski Add an explicit error return to lookup/range
rzezeski authored May 27, 2011
208 lists:flatten(lists:reverse(Acc));
209 {error, Ref, Reason} ->
210 {error, Reason}
1a0280a @rzezeski Merge in `lookup` and `range`
rzezeski authored May 18, 2011
211 after
212 ?LOOKUP_TIMEOUT ->
213 throw(lookup_timeout)
214 end.
Something went wrong with that request. Please try again.