Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
Use ets:select/2 to retrieve shards by name
The result of mem3_shards:for_db/1 on databases with high q values can
be very large, resulting in suboptimal performance for high-volume
callers.

mem3_sync_event_listener is only interested in a small subset of the
result of mem3_shards:for_db/1; moving this filter in to an ets:select/2
call improves performance significantly.

COUCHDB-2984
  • Loading branch information
b20n committed May 9, 2016
1 parent d5e0a4a commit 130efcd6b0b6f9fa6403e131586dfbf003643074
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 5 deletions.
@@ -21,6 +21,7 @@

-export([start_link/0]).
-export([for_db/1, for_db/2, for_docid/2, for_docid/3, get/3, local/1, fold/2]).
-export([for_shard_name/1]).
-export([set_max_size/1]).

-record(st, {
@@ -94,6 +95,42 @@ for_docid(DbName, DocId, Options) ->
false -> mem3_util:downcast(Shards)
end.

for_shard_name(ShardName) ->
for_shard_name(ShardName, []).

for_shard_name(ShardName, Options) ->
DbName = mem3:dbname(ShardName),
ShardHead = #shard{
name = ShardName,
node = '_',
dbname = DbName,
range = '_',
ref = '_'
},
OrderedShardHead = #ordered_shard{
name = ShardName,
node = '_',
dbname = DbName,
range = '_',
ref = '_',
order = '_'
},
ShardSpec = {ShardHead, [], ['$_']},
OrderedShardSpec = {OrderedShardHead, [], ['$_']},
Shards = try ets:select(?SHARDS, [ShardSpec, OrderedShardSpec]) of
[] ->
filter_shards_by_name(ShardName, load_shards_from_disk(DbName));
Else ->
gen_server:cast(?MODULE, {cache_hit, DbName}),
Else
catch error:badarg ->
filter_shards_by_name(ShardName, load_shards_from_disk(DbName))
end,
case lists:member(ordered, Options) of
true -> Shards;
false -> mem3_util:downcast(Shards)
end.

get(DbName, Node, Range) ->
Res = lists:foldl(fun(#shard{node=N, range=R}=S, Acc) ->
case {N, R} of
@@ -360,3 +397,15 @@ cache_clear(St) ->
true = ets:delete_all_objects(?SHARDS),
true = ets:delete_all_objects(?ATIMES),
St#st{cur_size=0}.

filter_shards_by_name(Name, Shards) ->
filter_shards_by_name(Name, [], Shards).

filter_shards_by_name(_, Matches, []) ->
Matches;
filter_shards_by_name(Name, Matches, [#ordered_shard{name=Name}=S|Ss]) ->
filter_shards_by_name(Name, [S|Matches], Ss);
filter_shards_by_name(Name, Matches, [#shard{name=Name}=S|Ss]) ->
filter_shards_by_name(Name, [S|Matches], Ss);
filter_shards_by_name(Name, Matches, [_|Ss]) ->
filter_shards_by_name(Name, Matches, Ss).
@@ -207,13 +207,18 @@ maybe_push_shards(St) ->
end.

push_shard(ShardName) ->
try mem3:shards(mem3:dbname(ShardName)) of
try mem3_shards:for_shard_name(ShardName) of
Shards ->
Targets = [S || #shard{node=N, name=Name} = S <- Shards,
N =/= node(), Name =:= ShardName],
Live = nodes(),
[mem3_sync:push(ShardName,N) || #shard{node=N} <- Targets,
lists:member(N, Live)]
lists:foreach(
fun(#shard{node=N}) ->
case lists:member(N, Live) of
true -> mem3_sync:push(ShardName, N);
false -> ok
end
end,
Shards
)
catch error:database_does_not_exist ->
ok
end.

0 comments on commit 130efcd

Please sign in to comment.