Permalink
Browse files

Merge branch 'master' into hanoidb-integration

  • Loading branch information...
2 parents 182f59e + d4f209f commit 858afddd5183fffe01e705be5e469d106971c2bd Gregory Burd committed Nov 15, 2012
Showing with 2,943 additions and 4,401 deletions.
  1. +1 −0 .travis.yml
  2. +20 −0 include/riak_kv_mrc_sink.hrl
  3. +7 −5 include/riak_kv_vnode.hrl
  4. BIN rebar
  5. +1 −1 rebar.config
  6. +0 −46 src/lk.erl
  7. +50 −307 src/riak_client.erl
  8. +1 −4 src/riak_index.erl
  9. +2 −9 src/riak_kv.app.src
  10. +43 −15 src/riak_kv_app.erl
  11. +5 −7 src/riak_kv_bitcask_backend.erl
  12. +2 −1 src/riak_kv_bucket.erl
  13. +11 −34 src/riak_kv_buckets_fsm.erl
  14. +2 −2 src/riak_kv_console.erl
  15. +13 −71 src/riak_kv_delete.erl
  16. +131 −1 src/riak_kv_eleveldb_backend.erl
  17. +9 −11 src/riak_kv_encoding_migrate.erl
  18. +81 −0 src/riak_kv_fsm_timing.erl
  19. +14 −15 src/riak_kv_get_core.erl
  20. +116 −93 src/riak_kv_get_fsm.erl
  21. +43 −46 src/riak_kv_index_fsm.erl
  22. +0 −5 src/riak_kv_js_manager.erl
  23. +0 −125 src/riak_kv_keylister_legacy.erl
  24. +0 −48 src/riak_kv_keylister_legacy_sup.erl
  25. +0 −80 src/riak_kv_keylister_master.erl
  26. +18 −48 src/riak_kv_keys_fsm.erl
  27. +0 −284 src/riak_kv_keys_fsm_legacy.erl
  28. +0 −49 src/riak_kv_keys_fsm_legacy_sup.erl
  29. +5 −6 src/riak_kv_legacy_vnode.erl
  30. +0 −352 src/riak_kv_lru.erl
  31. +0 −262 src/riak_kv_map_master.erl
  32. +0 −301 src/riak_kv_map_phase.erl
  33. +0 −311 src/riak_kv_mapper.erl
  34. +0 −48 src/riak_kv_mapper_sup.erl
  35. +0 −80 src/riak_kv_mapred_cache.erl
  36. +0 −75 src/riak_kv_mapred_planner.erl
  37. +0 −214 src/riak_kv_mapred_query.erl
  38. +1 −1 src/riak_kv_mapred_term.erl
  39. +4 −4 src/riak_kv_mapreduce.erl
  40. +36 −2 src/riak_kv_memory_backend.erl
  41. +271 −28 src/riak_kv_mrc_pipe.erl
  42. +434 −0 src/riak_kv_mrc_sink.erl
  43. +83 −0 src/riak_kv_mrc_sink_sup.erl
  44. +1 −0 src/riak_kv_multi_backend.erl
  45. +96 −151 src/riak_kv_pb_mapred.erl
  46. +0 −35 src/riak_kv_phase_proto.erl
  47. +54 −25 src/riak_kv_pipe_get.erl
  48. +36 −9 src/riak_kv_pipe_index.erl
  49. +27 −6 src/riak_kv_pipe_listkeys.erl
  50. +6 −28 src/riak_kv_put_fsm.erl
  51. +0 −112 src/riak_kv_reduce_phase.erl
  52. +156 −328 src/riak_kv_stat.erl
  53. +396 −0 src/riak_kv_stat_bc.erl
  54. +5 −31 src/riak_kv_sup.erl
  55. +183 −7 src/riak_kv_test_util.erl
  56. +1 −13 src/riak_kv_util.erl
  57. +62 −94 src/riak_kv_vnode.erl
  58. +11 −16 src/riak_kv_wm_link_walker.erl
  59. +75 −229 src/riak_kv_wm_mapred.erl
  60. +3 −3 src/riak_kv_wm_object.erl
  61. +20 −5 src/riak_kv_wm_props.erl
  62. +2 −1 src/riak_kv_wm_utils.erl
  63. +1 −1 src/riak_object.erl
  64. +5 −2 test/backend_eqc.erl
  65. +1 −0 test/fsm_eqc_util.erl
  66. +23 −21 test/get_fsm_qc.erl
  67. +61 −138 test/keys_fsm_eqc.erl
  68. +314 −155 test/mapred_test.erl
View
@@ -1,4 +1,5 @@
language: erlang
+script: (rebar compile && rebar eunit skip_deps=true) || (find . -name "*.log" -print -exec cat \{\} \; && sh -c "exit 1")
notifications:
webhooks: http://basho-engbot.herokuapp.com/travis?key=ad9a6e51d706903e1fd0963c7b8e064b93e85b56
email: eng@basho.com
@@ -0,0 +1,20 @@
+%% used to communicate from riak_kv_mrc_sink to riak_kv_wm_mapred and
+%% riak_kv_pb_mapred
+-record(kv_mrc_sink,
+ {
+ ref :: reference(), % the pipe ref
+ results :: [{PhaseId::integer(), Result::term()}],
+ logs :: [{PhaseId::integer(), Message::term()}],
+ done :: boolean()
+ }).
+
+%% used by riak_kv_mrc_sink:mapred_stream_sink
+-record(mrc_ctx,
+ {
+ ref :: reference(), % the pipe ref (so we don't have to dig)
+ pipe :: riak_pipe:pipe(),
+ sink :: {pid(), reference()}, % sink and monitor
+ sender :: {pid(), reference()}, % async sender and monitor
+ timer :: {reference(), reference()}, % timeout timer and pipe ref
+ keeps :: integer()
+ }).
@@ -16,10 +16,6 @@
req_id :: non_neg_integer(),
from :: term()}).
--record(riak_kv_listkeys_req_v1, {
- bucket :: binary(),
- req_id :: non_neg_integer()}).
-
-record(riak_kv_listkeys_req_v2, {
bucket :: binary()|'_'|tuple(),
req_id :: non_neg_integer(),
@@ -42,6 +38,12 @@
item_filter :: function(),
qry :: riak_index:query_def()}).
+%% same as _v1, but triggers ack-based backpressure
+-record(riak_kv_index_req_v2, {
+ bucket :: binary() | tuple(),
+ item_filter :: function(),
+ qry :: riak_index:query_def()}).
+
-record(riak_kv_vnode_status_req_v1, {}).
-record(riak_kv_delete_req_v1, {
@@ -63,7 +65,7 @@
-define(KV_MGET_REQ, #riak_kv_mget_req_v1).
-define(KV_LISTBUCKETS_REQ, #riak_kv_listbuckets_req_v1).
-define(KV_LISTKEYS_REQ, #riak_kv_listkeys_req_v4).
--define(KV_INDEX_REQ, #riak_kv_index_req_v1).
+-define(KV_INDEX_REQ, #riak_kv_index_req_v2).
-define(KV_VNODE_STATUS_REQ, #riak_kv_vnode_status_req_v1).
-define(KV_DELETE_REQ, #riak_kv_delete_req_v1).
-define(KV_MAP_REQ, #riak_kv_map_req_v1).
View
Binary file not shown.
View
@@ -1,6 +1,7 @@
{cover_enabled, true}.
{edoc_opts, [{preprocess, true}]}.
{erl_opts, [warnings_as_errors, {parse_transform, lager_transform}]}.
+{eunit_opts, [verbose]}.
{erl_first_files, [
"src/riak_kv_backend.erl",
@@ -9,7 +10,6 @@
{deps, [
{riak_core, ".*", {git, "git://github.com/basho/riak_core", "master"}},
- {luke, ".*", {git, "git://github.com/basho/luke", "master"}},
{erlang_js, ".*", {git, "git://github.com/basho/erlang_js", "master"}},
{bitcask, ".*", {git, "git://github.com/basho/bitcask", "master"}},
{merge_index, ".*", {git, "git://github.com/basho/merge_index",
View
@@ -1,46 +0,0 @@
-%% -------------------------------------------------------------------
-%%
-%% lk: Helper functions for list keys
-%%
-%% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved.
-%%
-%% This file is provided to you under the Apache License,
-%% Version 2.0 (the "License"); you may not use this file
-%% except in compliance with the License. You may obtain
-%% a copy of the License at
-%%
-%% http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing,
-%% software distributed under the License is distributed on an
-%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-%% KIND, either express or implied. See the License for the
-%% specific language governing permissions and limitations
-%% under the License.
-%%
-%% -------------------------------------------------------------------
--module(lk).
-
--export([fsm/1]).
-
-fsm(Bucket) ->
- ReqId = random:uniform(10000),
- Start = erlang:now(),
- riak_kv_keys_fsm:start_link(ReqId, Bucket, 60000, plain, 0.0001, self()),
- {ok, Count} = gather_fsm_results(ReqId, 0),
- End = erlang:now(),
- Ms = erlang:round(timer:now_diff(End, Start) / 1000),
- io:format("Found ~p keys in ~pms.~n", [Count, Ms]).
-
-gather_fsm_results(ReqId, Count) ->
- receive
- {ReqId, From, {keys, Keys}} ->
- riak_kv_keys_fsm:ack_keys(From),
- gather_fsm_results(ReqId, Count + length(Keys));
- {ReqId, {keys, Keys}} ->
- gather_fsm_results(ReqId, Count + length(Keys));
- {ReqId, done} ->
- {ok, Count}
- after 120000 ->
- {error, timeout}
- end.
Oops, something went wrong. Retry.

0 comments on commit 858afdd

Please sign in to comment.