Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merged from cliffmoon master

Signed-off-by: Jason Pellerin <jhp@jpdesk.(none)>
  • Loading branch information...
commit 7a70e0eb1dc83e9f31c2b6bdd2035da8b435ad3e 2 parents ef595bf + b815f42
Jason Pellerin authored
28 NOTES
View
@@ -3,3 +3,31 @@
- seeming instability in get/put as nodes join/leave?
- missing tests for node joining?
+
+- after activating a new node in a group, got an error when getting a key:
+
+
+Error in process <0.774.0> on node 'b5@localhost' with exit value: {{case_clause,{exit,{{function_clause,[{vector_clock,resolve,[not_found,{[{'b1@localhost',1.227209e+09},{'b2@localhost',1.227212e+09},{'b3@localhost',1.227212e+09},{'b4@localhost',1.227212e+09}],[<<50 bytes>>]}]},{mediator,internal_get,2},{mediator,'-handle_call/3-fun-0-',3}]},{gen_server...
+
+
+- is gossip necessary? it's very busy, can we instead just use erlang node
+ monitoring?
+
+
+ec2 baseline
+
+4 clients on 1 server, 4 nodes:
+
+[root@domU-12-31-38-00-A1-D8 pylibs]# PYTHONPATH=. ./tools/dbench_thrift.py -n 1000 -c 4
+. . . .
+4 client(s) 1000 request(s) 288.8384120.3s
+get avg: 19.2921230.3ms mean: 7.6100830.3ms 99.9: 128.2091140.3ms
+put avg: 52.9174800.3ms mean: 44.9030400.3ms 99.9: 192.3902030.3ms
+
+19 clients on 1 server, 4 nodes:
+
+[root@domU-12-31-38-00-A1-D8 pylibs]# PYTHONPATH=. ./tools/dbench_thrift.py -n 1000 -c 10
+. . . . . . . . . .
+10 client(s) 1000 request(s) 2450.8665540.3s
+get avg: 69.8761190.3ms mean: 63.1911750.3ms 99.9: 479.4890880.3ms
+put avg: 175.2105370.3ms mean: 164.6809580.3ms 99.9: 581.4800260.3ms
4 Rakefile
View
@@ -23,6 +23,10 @@ task :native do
ERLC_FLAGS = "+native #{ERLC_FLAGS}"
end
+task :profile do
+ ERLC_FLAGS = "-DPROF #{ERLC_FLAGS}"
+end
+
task :run do
sh %Q{erl -boot start_sasl +K true +A 128 -smp enable -pz ./ebin/ -sname local_console#{$$} -mnesia dir '"/tmp/mbd"' -noshell -run dynomite start}
end
15 elibs/lib_misc.erl
View
@@ -126,4 +126,17 @@ byte_size(Term) ->
listify(List) when is_list(List) ->
List;
-listify(El) -> [El].
+listify(El) -> [El].
+
+reverse_bits(V) when is_integer(V) ->
+ % swap odd and even bits
+ V1 = ((V bsr 1) band 16#55555555) bor (((V band 16#55555555) bsl 1) band 16#ffffffff),
+ % swap consecutive pairs
+ V2 = ((V1 bsr 2) band 16#33333333) bor (((V1 band 16#33333333) bsl 2) band 16#ffffffff),
+ % swap nibbles ...
+ V3 = ((V2 bsr 4) band 16#0F0F0F0F) bor (((V2 band 16#0F0F0F0F) bsl 4) band 16#ffffffff),
+ % swap bytes
+ V4 = ((V3 bsr 8) band 16#00FF00FF) bor (((V3 band 16#00FF00FF) bsl 8) band 16#ffffffff),
+ % swap 2-byte long pairs
+ ((V4 bsr 16) band 16#ffffffff) bor ((V4 bsl 16) band 16#ffffffff).
+
4 elibs/mediator.erl
View
@@ -205,9 +205,7 @@ internal_delete(Key, #mediator{config=Config}) ->
MapFun = fun(Server) ->
storage_server:delete({list_to_atom(lists:concat([storage_, Part])), Server}, Key, 10000)
end,
- Blah = pcall(MapFun, Servers, W),
- {Good, Bad} = Blah,
- % ok = Blah,
+ {Good, Bad} = pcall(MapFun, Servers, W),
if
length(Good) >= W -> {ok, length(Good)};
true -> {failure, error_message(Good, Bad, N, W)}
7 elibs/storage_server.erl
View
@@ -26,6 +26,8 @@
-include("etest/storage_server_test.erl").
-endif.
+-include("profile.hrl").
+
%%====================================================================
%% API
%%====================================================================
@@ -249,15 +251,20 @@ code_change(_OldVsn, State, _Extra) ->
internal_put(Key, Context, Values, Tree, Table, Module, State) ->
SKey = sanitize_key(Key),
%% ?debugFmt("Internal put key ~p module ~p", [SKey, Module]),
+ ?prof(dmerkle_update),
UpdatedTree = dmerkle:update(SKey, Values, Tree),
+ ?prof(dmerkle_update),
%% ?debugMsg("Tree updated"),
+ ?prof(put),
case catch Module:put(SKey, Context, Values, Table) of
{ok, ModifiedTable} ->
%% ?debugMsg("Table modified"),
+ ?prof(put),
stats_server:request(put, lib_misc:byte_size(Values)),
{reply, ok, State#storage{table=ModifiedTable,tree=UpdatedTree}};
Failure ->
%% ?debugFmt("failure ~p", [Failure]),
+ ?prof(put),
{reply, Failure, State}
end.
5 etest/lib_misc_test.erl
View
@@ -12,4 +12,7 @@ pmap_1_test() ->
[{0,0}] = pmap(fun(N) ->
N
end, L, 1).
-
+
+reverse_bits_test() ->
+ 3869426816 = reverse_bits(19088743),
+ 1458223569 = reverse_bits(2342344554).
33 pylibs/test/ec2/ec2_load_test.py
View
@@ -21,3 +21,36 @@
EC2_PUT_THRESHOLD -- If 99.9% of puts are not faster than this # of
milliseconds, the test fails
"""
+import os
+import sys
+import boto
+
+
+def load_test(conf=None):
+ if conf is None:
+ conf = configure()
+ instances = start_instances(conf)
+ try:
+ start_load(conf, instances)
+ wait(conf)
+ stats = collect_stats(conf, instances)
+ finally:
+ stop_instances(conf, instances)
+ evaluate_stats(conf, stats)
+
+
+def main():
+ conf = configure(sys.argv)
+ load_test(conf)
+
+
+def configure(argv=None):
+ if argv is None:
+ argv = []
+ # FIXME
+ env = os.environ
+ return options
+
+
+if __name__ == '__main__':
+ main()
2  pylibs/tools/load_thrift.py
View
@@ -92,6 +92,8 @@ def run(rq, host, port, sleep, minsize, maxsize):
def opts():
parser = OptionParser()
+ # FIXME num requests / time to run
+
parser.add_option('--host', dest='host', default='localhost',
action='store', help='Connect to host')
parser.add_option('-p', '--port', dest='port', default=9200,
Please sign in to comment.
Something went wrong with that request. Please try again.