diff --git a/.gitignore b/.gitignore index 81a9f638..6fa90851 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ Gemfile.lock vendor/erlang vendor/riak .ruby-version +tmp diff --git a/Rakefile b/Rakefile index ed36a8a9..c33d6eb2 100644 --- a/Rakefile +++ b/Rakefile @@ -70,3 +70,38 @@ RSpec::Core::RakeTask.new(:ci) do |spec| spec.rspec_opts = %w[--profile] end task :default => :ci + + +desc "Generate Protocol Buffers message definitions from riak_pb" +task :pb_defs => 'beefcake:pb_defs' + +namespace :beefcake do + task :pb_defs => 'lib/riak/client/beefcake/messages.rb' + + task :clean do + sh "rm -rf tmp/riak_pb" + sh "rm -rf tmp/riak_kv.pb.rb tmp/riak_search.pb.rb tmp/riak_yokozuna.pb.rb" + end + + file 'lib/riak/client/beefcake/messages.rb' => %w{tmp/riak_kv.pb.rb tmp/riak_search.pb.rb tmp/riak_yokozuna.pb.rb} do |t| + sh "cat lib/riak/client/beefcake/header tmp/riak.pb.rb #{t.prerequisites.join ' '} lib/riak/client/beefcake/footer > #{t.name}" + end + + file 'tmp/riak_kv.pb.rb' => 'tmp/riak_pb' do |t| + sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/riak_kv.proto" + end + + file 'tmp/riak_search.pb.rb' => 'tmp/riak_pb' do |t| + sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/riak_search.proto" + end + + file 'tmp/riak_yokozuna.pb.rb' => 'tmp/riak_pb' do |t| + sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/riak_yokozuna.proto" + end + + directory 'tmp/riak_pb' do + cd 'tmp' do + sh "git clone -b develop https://github.com/basho/riak_pb.git" + end + end +end diff --git a/lib/riak/client.rb b/lib/riak/client.rb index 2092eff2..ee906368 100644 --- a/lib/riak/client.rb +++ b/lib/riak/client.rb @@ -8,6 +8,7 @@ require 'riak/client/decaying' require 'riak/client/node' require 'riak/client/search' +require 'riak/client/yokozuna' require 'riak/client/http_backend' require 'riak/client/net_http_backend' require 'riak/client/excon_backend' diff --git a/lib/riak/client/beefcake/footer b/lib/riak/client/beefcake/footer new file mode 100644 index 00000000..f8fdd785 --- /dev/null +++ b/lib/riak/client/beefcake/footer @@ -0,0 +1,4 @@ + + end + end +end diff --git a/lib/riak/client/beefcake/header b/lib/riak/client/beefcake/header new file mode 100644 index 00000000..960f1bc1 --- /dev/null +++ b/lib/riak/client/beefcake/header @@ -0,0 +1,6 @@ +require 'beefcake' + +module Riak + class Client + # @private + class BeefcakeProtobuffsBackend diff --git a/lib/riak/client/beefcake/message_codes.rb b/lib/riak/client/beefcake/message_codes.rb index 597f1275..028f41b7 100644 --- a/lib/riak/client/beefcake/message_codes.rb +++ b/lib/riak/client/beefcake/message_codes.rb @@ -39,6 +39,13 @@ module BeefcakeMessageCodes :CounterUpdateResp => 51, :CounterGetReq => 52, :CounterGetResp => 53, + :YokozunaIndexGetReq => 54, + :YokozunaIndexGetResp => 55, + :YokozunaIndexPutReq => 56, + :YokozunaIndexDeleteReq => 57, + :YokozunaSchemaGetReq => 58, + :YokozunaSchemaGetResp => 59, + :YokozunaSchemaPutReq => 60, } CODE_TO_MESSAGE = MESSAGE_TO_CODE.invert diff --git a/lib/riak/client/beefcake/message_overlay.rb b/lib/riak/client/beefcake/message_overlay.rb new file mode 100644 index 00000000..3c2caac4 --- /dev/null +++ b/lib/riak/client/beefcake/message_overlay.rb @@ -0,0 +1,49 @@ +module Riak + class Client + # @private + class BeefcakeProtobuffsBackend + class RpbIndexReq + module IndexQueryType + EQ = 0 + RANGE = 1 + end + end + + class RpbBucketProps + + # "repeated" elements with zero items are indistinguishable + # from a nil, so we have to manage has_precommit/has_postcommit + # flags. + def precommit=(newval) + @precommit = newval + @has_precommit = !!newval + end + + def has_precommit=(newval) + @has_precommit = newval + @precommit ||= [] if newval + end + + def postcommit=(newval) + @postcommit = newval + @has_postcommit = !!newval + end + + def has_postcommit=(newval) + @has_postcommit = newval + @postcommit ||= [] if newval + end + end + + class RpbSearchDoc + # rebuild the fields instance method since the + # generated :fields field overwrote this + def fields + self.class.fields + end + repeated :properties, RpbPair, 1 + end + + end + end +end diff --git a/lib/riak/client/beefcake/messages.rb b/lib/riak/client/beefcake/messages.rb index e85936fd..074caa0e 100644 --- a/lib/riak/client/beefcake/messages.rb +++ b/lib/riak/client/beefcake/messages.rb @@ -4,382 +4,579 @@ module Riak class Client # @private class BeefcakeProtobuffsBackend - # Embedded messages - class RpbPair - include Beefcake::Message - required :key, :bytes, 1 - optional :value, :bytes, 2 - end - - # module-function pair for commit hooks and other properties that take - # functions - class RpbModFun - include Beefcake::Message - - required :module, :bytes, 1 - required :function, :bytes, 2 - end - - class RpbCommitHook - include Beefcake::Message - - optional :modfun, RpbModFun, 1 - optional :name, :bytes, 2 - end - - class RpbBucketProps - include Beefcake::Message - - # riak_core_app - optional :n_val, :uint32, 1 - optional :allow_mult, :bool, 2 - optional :last_write_wins, :bool, 3 - - # riak_core values with special handling, see below - repeated :precommit, RpbCommitHook, 4 - optional :has_precommit, :bool, 5, :default => false - repeated :postcommit, RpbCommitHook, 6 - optional :has_postcommit, :bool, 7, :default => false - - optional :chash_keyfun, RpbModFun, 8 - - # riak_kv_app - optional :linkfun, RpbModFun, 9 - optional :old_vclock, :uint32, 10 - optional :young_vclock, :uint32, 11 - optional :big_vclock, :uint32, 12 - optional :small_vclock, :uint32, 13 - optional :pr, :uint32, 14 - optional :r, :uint32, 15 - optional :w, :uint32, 16 - optional :pw, :uint32, 17 - optional :dw, :uint32, 18 - optional :rw, :uint32, 19 - optional :basic_quorum, :bool, 20 - optional :notfound_ok, :bool, 21 - - # riak_kv_multi_backend - optional :backend, :bytes, 22 - - # riak_search bucket fixup - optional :search, :bool, 23 - - module RpbReplMode - FALSE = 0 - REALTIME = 1 - FULLSYNC = 2 - TRUE = 3 - end - - optional :repl, RpbReplMode, 24 - - # "repeated" elements with zero items are indistinguishable - # from a nil, so we have to manage has_precommit/has_postcommit - # flags. - def precommit=(newval) - @precommit = newval - @has_precommit = !!newval - end - - def has_precommit=(newval) - @has_precommit = newval - @precommit ||= [] if newval - end - - def postcommit=(newval) - @postcommit = newval - @has_postcommit = !!newval - end - - def has_postcommit=(newval) - @has_postcommit = newval - @postcommit ||= [] if newval - end - end - - class RpbLink - include Beefcake::Message - optional :bucket, :bytes, 1 - optional :key, :bytes, 2 - optional :tag, :bytes, 3 - end - - class RpbContent - include Beefcake::Message - required :value, :bytes, 1 - optional :content_type, :bytes, 2 - optional :charset, :bytes, 3 - optional :content_encoding, :bytes, 4 - optional :vtag, :bytes, 5 - repeated :links, RpbLink, 6 - optional :last_mod, :uint32, 7 - optional :last_mod_usecs, :uint32, 8 - repeated :usermeta, RpbPair, 9 - repeated :indexes, RpbPair, 10 - end - - # Primary messages - class RpbErrorResp - include Beefcake::Message - required :errmsg, :bytes, 1 - required :errcode, :uint32, 2 - end - - class RpbGetClientIdResp - include Beefcake::Message - required :client_id, :bytes, 1 - end - - class RpbSetClientIdReq - include Beefcake::Message - required :client_id, :bytes, 1 - end - - class RpbGetServerInfoResp - include Beefcake::Message - optional :node, :bytes, 1 - optional :server_version, :bytes, 2 - end - - class RpbGetReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :key, :bytes, 2 - optional :r, :uint32, 3 - optional :pr, :uint32, 4 - optional :basic_quorum, :bool, 5 - optional :notfound_ok, :bool, 6 - optional :if_modified, :bytes, 7 - optional :head, :bool, 8 - optional :deletedvclock, :bool, 9 - optional :timeout, :uint32, 10 - optional :sloppy_quorum, :bool, 11 - optional :n_val, :uint32, 12 - end - - class RpbGetResp - include Beefcake::Message - repeated :content, RpbContent, 1 - optional :vclock, :bytes, 2 - optional :unchanged, :bool, 3 - end - - class RpbPutReq - include Beefcake::Message - required :bucket, :bytes, 1 - optional :key, :bytes, 2 - optional :vclock, :bytes, 3 - required :content, RpbContent, 4 - optional :w, :uint32, 5 - optional :dw, :uint32, 6 - optional :returnbody, :bool, 7 - optional :pw, :uint32, 8 - optional :if_not_modified, :bool, 9 - optional :if_none_match, :bool, 10 - optional :return_head, :bool, 11 - optional :timeout, :uint32, 12 - optional :asis, :bool, 13 - optional :sloppy_quorum, :bool, 14 - optional :n_val, :uint32, 15 - end - - class RpbPutResp - include Beefcake::Message - repeated :content, RpbContent, 1 - optional :vclock, :bytes, 2 - optional :key, :bytes, 3 - end - - class RpbDelReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :key, :bytes, 2 - optional :rw, :uint32, 3 - optional :vclock, :bytes, 4 - optional :r, :uint32, 5 - optional :w, :uint32, 6 - optional :pr, :uint32, 7 - optional :pw, :uint32, 8 - optional :dw, :uint32, 9 - optional :timeout, :uint32, 10 - optional :sloppy_quorum, :bool, 11 - optional :n_val, :uint32, 12 - end - - class RpbListBucketsReq - include Beefcake::Message - optional :timeout, :uint32, 1 - optional :stream, :bool, 2 - end - - class RpbListBucketsResp - include Beefcake::Message - repeated :buckets, :bytes, 1 - optional :done, :bool, 2 - end - - class RpbListKeysReq - include Beefcake::Message - required :bucket, :bytes, 1 - optional :timeout, :uint32, 2 - end - - class RpbListKeysResp - include Beefcake::Message - repeated :keys, :bytes, 1 - optional :done, :bool, 2 - end - - class RpbGetBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - end - - class RpbGetBucketResp - include Beefcake::Message - required :props, RpbBucketProps, 1 - end - - class RpbSetBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :props, RpbBucketProps, 2 - end - - class RpbResetBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - end - - class RpbMapRedReq - include Beefcake::Message - required :request, :bytes, 1 - required :content_type, :bytes, 2 - end - - class RpbMapRedResp - include Beefcake::Message - optional :phase, :uint32, 1 - optional :response, :bytes, 2 - optional :done, :bool, 3 - end - - class RpbIndexReq - include Beefcake::Message - module IndexQueryType - EQ = 0 - RANGE = 1 - end - - required :bucket, :bytes, 1 - required :index, :bytes, 2 - required :qtype, IndexQueryType, 3 - optional :key, :bytes, 4 - optional :range_min, :bytes, 5 - optional :range_max, :bytes, 6 - optional :return_terms, :bool, 7 - optional :stream, :bool, 8 - optional :max_results, :uint32, 9 - optional :continuation, :bytes, 10 - optional :timeout, :uint32, 11 - end - - class RpbIndexResp - include Beefcake::Message - repeated :keys, :bytes, 1 - repeated :results, RpbPair, 2 - optional :continuation, :bytes, 3 - optional :done, :bool, 4 - end - - class RpbSearchDoc - include Beefcake::Message - # We have to name this differently than the .proto file does - # because Beefcake uses 'fields' as an instance method. - repeated :properties, RpbPair, 1 - end - - class RpbSearchQueryReq - include Beefcake::Message - required :q, :bytes, 1 - required :index, :bytes, 2 - optional :rows, :uint32, 3 - optional :start, :uint32, 4 - optional :sort, :bytes, 5 - optional :filter, :bytes, 6 - optional :df, :bytes, 7 - optional :op, :bytes, 8 - repeated :fl, :bytes, 9 - optional :presort, :bytes, 10 - end - - class RpbSearchQueryResp - include Beefcake::Message - repeated :docs, RpbSearchDoc, 1, :default => [] - optional :max_score, :float, 2 - optional :num_found, :uint32, 3 - end - - class RpbResetBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - end - - class RpbCSBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :start_key, :bytes, 2 - optional :end_key, :bytes, 3 - optional :start_incl, :bool, 4, default: true - optional :end_incl, :bool, 5, default: false - optional :continuation, :bytes, 6 - optional :max_results, :uint32, 7 - optional :timeout, :uint32, 8 - end - - class RpbIndexObject - include Beefcake::Message - required :key, :bytes, 1 - required :object, RpbGetResp, 2 - end - - class RpbCSBucketResp - include Beefcake::Message - repeated :objects, RpbIndexObject, 1 - optional :continuation, :bytes, 2 - optional :done, :bool, 3 - end - - class RpbCounterUpdateReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :key, :bytes, 2 - required :amount, :sint64, 3 - optional :w, :uint32, 4 - optional :dw, :uint32, 5 - optional :pw, :uint32, 6 - optional :returnvalue, :bool, 7 - end - - class RpbCounterUpdateResp - include Beefcake::Message - optional :value, :sint64, 1 - end - - class RpbCounterGetReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :key, :bytes, 2 - optional :r, :uint32, 3 - optional :pr, :uint32, 4 - optional :basic_quorum, :bool, 5 - optional :notfound_ok, :bool, 6 - end - - class RpbCounterGetResp - include Beefcake::Message - optional :value, :sint64, 1 - end +## Generated from riak.proto for +require "beefcake" + + +class RpbErrorResp + include Beefcake::Message +end + + +class RpbGetServerInfoResp + include Beefcake::Message +end + + +class RpbPair + include Beefcake::Message +end + + +class RpbGetBucketReq + include Beefcake::Message +end + + +class RpbGetBucketResp + include Beefcake::Message +end + + +class RpbSetBucketReq + include Beefcake::Message +end + + +class RpbResetBucketReq + include Beefcake::Message +end + + +class RpbModFun + include Beefcake::Message +end + + +class RpbCommitHook + include Beefcake::Message +end + + +class RpbBucketProps + include Beefcake::Message +end + + +class RpbErrorResp + required :errmsg, :bytes, 1 + required :errcode, :uint32, 2 +end + +class RpbGetServerInfoResp + optional :node, :bytes, 1 + optional :server_version, :bytes, 2 +end + +class RpbPair + required :key, :bytes, 1 + optional :value, :bytes, 2 +end + +class RpbGetBucketReq + required :bucket, :bytes, 1 +end + +class RpbGetBucketResp + required :props, RpbBucketProps, 1 +end + +class RpbSetBucketReq + required :bucket, :bytes, 1 + required :props, RpbBucketProps, 2 +end + +class RpbResetBucketReq + required :bucket, :bytes, 1 +end + +class RpbModFun + required :module, :bytes, 1 + required :function, :bytes, 2 +end + +class RpbCommitHook + optional :modfun, RpbModFun, 1 + optional :name, :bytes, 2 +end + +class RpbBucketProps + module RpbReplMode + FALSE = 0 + REALTIME = 1 + FULLSYNC = 2 + TRUE = 3 + end + optional :n_val, :uint32, 1 + optional :allow_mult, :bool, 2 + optional :last_write_wins, :bool, 3 + repeated :precommit, RpbCommitHook, 4 + optional :has_precommit, :bool, 5, :default => false + repeated :postcommit, RpbCommitHook, 6 + optional :has_postcommit, :bool, 7, :default => false + optional :chash_keyfun, RpbModFun, 8 + optional :linkfun, RpbModFun, 9 + optional :old_vclock, :uint32, 10 + optional :young_vclock, :uint32, 11 + optional :big_vclock, :uint32, 12 + optional :small_vclock, :uint32, 13 + optional :pr, :uint32, 14 + optional :r, :uint32, 15 + optional :w, :uint32, 16 + optional :pw, :uint32, 17 + optional :dw, :uint32, 18 + optional :rw, :uint32, 19 + optional :basic_quorum, :bool, 20 + optional :notfound_ok, :bool, 21 + optional :backend, :bytes, 22 + optional :search, :bool, 23 + optional :repl, RpbBucketProps::RpbReplMode, 24 + optional :yz_index, :bytes, 25 +end +## Generated from riak_kv.proto for +require "beefcake" + + +class RpbGetClientIdResp + include Beefcake::Message +end + + +class RpbSetClientIdReq + include Beefcake::Message +end + + +class RpbGetReq + include Beefcake::Message +end + + +class RpbGetResp + include Beefcake::Message +end + + +class RpbPutReq + include Beefcake::Message +end + + +class RpbPutResp + include Beefcake::Message +end + + +class RpbDelReq + include Beefcake::Message +end + + +class RpbListBucketsReq + include Beefcake::Message +end + + +class RpbListBucketsResp + include Beefcake::Message +end + + +class RpbListKeysReq + include Beefcake::Message +end + + +class RpbListKeysResp + include Beefcake::Message +end + + +class RpbMapRedReq + include Beefcake::Message +end + + +class RpbMapRedResp + include Beefcake::Message +end + + +class RpbIndexReq + include Beefcake::Message +end + + +class RpbIndexResp + include Beefcake::Message +end + + +class RpbCSBucketReq + include Beefcake::Message +end + + +class RpbCSBucketResp + include Beefcake::Message +end + + +class RpbIndexObject + include Beefcake::Message +end + + +class RpbContent + include Beefcake::Message +end + + +class RpbLink + include Beefcake::Message +end + + +class RpbCounterUpdateReq + include Beefcake::Message +end + + +class RpbCounterUpdateResp + include Beefcake::Message +end + + +class RpbCounterGetReq + include Beefcake::Message +end + + +class RpbCounterGetResp + include Beefcake::Message +end + + +class RpbGetClientIdResp + required :client_id, :bytes, 1 +end + +class RpbSetClientIdReq + required :client_id, :bytes, 1 +end + +class RpbGetReq + required :bucket, :bytes, 1 + required :key, :bytes, 2 + optional :r, :uint32, 3 + optional :pr, :uint32, 4 + optional :basic_quorum, :bool, 5 + optional :notfound_ok, :bool, 6 + optional :if_modified, :bytes, 7 + optional :head, :bool, 8 + optional :deletedvclock, :bool, 9 + optional :timeout, :uint32, 10 + optional :sloppy_quorum, :bool, 11 + optional :n_val, :uint32, 12 +end + +class RpbGetResp + repeated :content, RpbContent, 1 + optional :vclock, :bytes, 2 + optional :unchanged, :bool, 3 +end + +class RpbPutReq + required :bucket, :bytes, 1 + optional :key, :bytes, 2 + optional :vclock, :bytes, 3 + required :content, RpbContent, 4 + optional :w, :uint32, 5 + optional :dw, :uint32, 6 + optional :return_body, :bool, 7 + optional :pw, :uint32, 8 + optional :if_not_modified, :bool, 9 + optional :if_none_match, :bool, 10 + optional :return_head, :bool, 11 + optional :timeout, :uint32, 12 + optional :asis, :bool, 13 + optional :sloppy_quorum, :bool, 14 + optional :n_val, :uint32, 15 +end + +class RpbPutResp + repeated :content, RpbContent, 1 + optional :vclock, :bytes, 2 + optional :key, :bytes, 3 +end + +class RpbDelReq + required :bucket, :bytes, 1 + required :key, :bytes, 2 + optional :rw, :uint32, 3 + optional :vclock, :bytes, 4 + optional :r, :uint32, 5 + optional :w, :uint32, 6 + optional :pr, :uint32, 7 + optional :pw, :uint32, 8 + optional :dw, :uint32, 9 + optional :timeout, :uint32, 10 + optional :sloppy_quorum, :bool, 11 + optional :n_val, :uint32, 12 +end + +class RpbListBucketsReq + optional :timeout, :uint32, 1 + optional :stream, :bool, 2 +end + +class RpbListBucketsResp + repeated :buckets, :bytes, 1 + optional :done, :bool, 2 +end + +class RpbListKeysReq + required :bucket, :bytes, 1 + optional :timeout, :uint32, 2 +end + +class RpbListKeysResp + repeated :keys, :bytes, 1 + optional :done, :bool, 2 +end + +class RpbMapRedReq + required :request, :bytes, 1 + required :content_type, :bytes, 2 +end + +class RpbMapRedResp + optional :phase, :uint32, 1 + optional :response, :bytes, 2 + optional :done, :bool, 3 +end + +class RpbIndexReq + module IndexQueryType + eq = 0 + range = 1 + end + required :bucket, :bytes, 1 + required :index, :bytes, 2 + required :qtype, RpbIndexReq::IndexQueryType, 3 + optional :key, :bytes, 4 + optional :range_min, :bytes, 5 + optional :range_max, :bytes, 6 + optional :return_terms, :bool, 7 + optional :stream, :bool, 8 + optional :max_results, :uint32, 9 + optional :continuation, :bytes, 10 + optional :timeout, :uint32, 11 +end + +class RpbIndexResp + repeated :keys, :bytes, 1 + repeated :results, RpbPair, 2 + optional :continuation, :bytes, 3 + optional :done, :bool, 4 +end + +class RpbCSBucketReq + required :bucket, :bytes, 1 + required :start_key, :bytes, 2 + optional :end_key, :bytes, 3 + optional :start_incl, :bool, 4, :default => true + optional :end_incl, :bool, 5, :default => false + optional :continuation, :bytes, 6 + optional :max_results, :uint32, 7 + optional :timeout, :uint32, 8 +end + +class RpbCSBucketResp + repeated :objects, RpbIndexObject, 1 + optional :continuation, :bytes, 2 + optional :done, :bool, 3 +end + +class RpbIndexObject + required :key, :bytes, 1 + required :object, RpbGetResp, 2 +end + +class RpbContent + required :value, :bytes, 1 + optional :content_type, :bytes, 2 + optional :charset, :bytes, 3 + optional :content_encoding, :bytes, 4 + optional :vtag, :bytes, 5 + repeated :links, RpbLink, 6 + optional :last_mod, :uint32, 7 + optional :last_mod_usecs, :uint32, 8 + repeated :usermeta, RpbPair, 9 + repeated :indexes, RpbPair, 10 + optional :deleted, :bool, 11 +end + +class RpbLink + optional :bucket, :bytes, 1 + optional :key, :bytes, 2 + optional :tag, :bytes, 3 +end + +class RpbCounterUpdateReq + required :bucket, :bytes, 1 + required :key, :bytes, 2 + required :amount, :sint64, 3 + optional :w, :uint32, 4 + optional :dw, :uint32, 5 + optional :pw, :uint32, 6 + optional :returnvalue, :bool, 7 +end + +class RpbCounterUpdateResp + optional :value, :sint64, 1 +end + +class RpbCounterGetReq + required :bucket, :bytes, 1 + required :key, :bytes, 2 + optional :r, :uint32, 3 + optional :pr, :uint32, 4 + optional :basic_quorum, :bool, 5 + optional :notfound_ok, :bool, 6 +end + +class RpbCounterGetResp + optional :value, :sint64, 1 +end +## Generated from riak_search.proto for +require "beefcake" + + +class RpbSearchDoc + include Beefcake::Message +end + + +class RpbSearchQueryReq + include Beefcake::Message +end + + +class RpbSearchQueryResp + include Beefcake::Message +end + + +class RpbSearchDoc + repeated :fields, RpbPair, 1 +end + +class RpbSearchQueryReq + required :q, :bytes, 1 + required :index, :bytes, 2 + optional :rows, :uint32, 3 + optional :start, :uint32, 4 + optional :sort, :bytes, 5 + optional :filter, :bytes, 6 + optional :df, :bytes, 7 + optional :op, :bytes, 8 + repeated :fl, :bytes, 9 + optional :presort, :bytes, 10 +end + +class RpbSearchQueryResp + repeated :docs, RpbSearchDoc, 1 + optional :max_score, :float, 2 + optional :num_found, :uint32, 3 +end +## Generated from riak_yokozuna.proto for +require "beefcake" + + +class RpbYokozunaIndex + include Beefcake::Message +end + + +class RpbYokozunaIndexGetReq + include Beefcake::Message +end + + +class RpbYokozunaIndexGetResp + include Beefcake::Message +end + + +class RpbYokozunaIndexPutReq + include Beefcake::Message +end + + +class RpbYokozunaIndexDeleteReq + include Beefcake::Message +end + + +class RpbYokozunaSchema + include Beefcake::Message +end + + +class RpbYokozunaSchemaPutReq + include Beefcake::Message +end + + +class RpbYokozunaSchemaGetReq + include Beefcake::Message +end + + +class RpbYokozunaSchemaGetResp + include Beefcake::Message +end + + +class RpbYokozunaIndex + required :name, :bytes, 1 + optional :schema, :bytes, 2 +end + +class RpbYokozunaIndexGetReq + optional :name, :bytes, 1 +end + +class RpbYokozunaIndexGetResp + repeated :index, RpbYokozunaIndex, 1 +end + +class RpbYokozunaIndexPutReq + required :index, RpbYokozunaIndex, 1 +end + +class RpbYokozunaIndexDeleteReq + required :name, :bytes, 1 +end + +class RpbYokozunaSchema + required :name, :bytes, 1 + optional :content, :bytes, 2 +end + +class RpbYokozunaSchemaPutReq + required :schema, RpbYokozunaSchema, 1 +end + +class RpbYokozunaSchemaGetReq + required :name, :bytes, 1 +end + +class RpbYokozunaSchemaGetResp + required :schema, RpbYokozunaSchema, 1 +end + end end end diff --git a/lib/riak/client/beefcake_protobuffs_backend.rb b/lib/riak/client/beefcake_protobuffs_backend.rb index 94ed84b8..706bb08c 100644 --- a/lib/riak/client/beefcake_protobuffs_backend.rb +++ b/lib/riak/client/beefcake_protobuffs_backend.rb @@ -11,6 +11,7 @@ def self.configured? begin require 'beefcake' require 'riak/client/beefcake/messages' + require 'riak/client/beefcake/message_overlay' require 'riak/client/beefcake/object_methods' true rescue LoadError, NameError @@ -210,6 +211,44 @@ def search(index, query, options={}) decode_response end + def create_search_index(name, schema=nil) + index = RpbYokozunaIndex.new(:name => name, :schema => schema) + req = RpbYokozunaIndexPutReq.new(:index => index) + write_protobuff(:YokozunaIndexPutReq, req) + decode_response + end + + def get_search_index(name) + req = RpbYokozunaIndexGetReq.new(:name => name) + write_protobuff(:YokozunaIndexGetReq, req) + resp = decode_response + if resp.index && Array === resp + resp.index.map{|index| {:name => index.name, :schema => index.schema} } + else + resp + end + end + + def delete_search_index(name) + req = RpbYokozunaIndexDeleteReq.new(:name => name) + write_protobuff(:YokozunaIndexDeleteReq, req) + decode_response + end + + def create_search_schema(name, content) + schema = RpbYokozunaSchema.new(:name => name, :content => content) + req = RpbYokozunaSchemaPutReq.new(:schema => schema) + write_protobuff(:YokozunaSchemaPutReq, req) + decode_response + end + + def get_search_schema(name) + req = RpbYokozunaSchemaGetReq.new(:name => name) + write_protobuff(:YokozunaSchemaGetReq, req) + resp = decode_response + resp.schema ? resp.schema : resp + end + private def write_protobuff(code, message) encoded = message.encode @@ -234,7 +273,9 @@ def decode_response(*args) :ListKeysResp, :IndexResp [] - when :GetResp + when :GetResp, + :YokozunaIndexGetResp, + :YokozunaSchemaGetResp raise Riak::ProtobuffsFailedRequest.new(:not_found, t('not_found')) when :CounterGetResp, :CounterUpdateResp @@ -286,6 +327,10 @@ def decode_response(*args) when :CounterGetResp res = RpbCounterGetResp.decode message res.value || 0 + when :YokozunaIndexGetResp + res = RpbYokozunaIndexGetResp.decode message + when :YokozunaSchemaGetResp + res = RpbYokozunaSchemaGetResp.decode message end end rescue SystemCallError, SocketError => e diff --git a/lib/riak/client/yokozuna.rb b/lib/riak/client/yokozuna.rb new file mode 100644 index 00000000..206148e8 --- /dev/null +++ b/lib/riak/client/yokozuna.rb @@ -0,0 +1,52 @@ +module Riak + class Client + def create_search_index(name, schema=nil) + raise ArgumentError, t("zero_length_index") if name.nil? || name.empty? + backend do |b| + b.create_search_index(name, schema) + end + true + end + + def get_search_index(name) + raise ArgumentError, t("zero_length_index") if name.nil? || name.empty? + resp = [] + backend do |b| + resp = b.get_search_index(name) + end + resp.index && Array === resp.index ? resp.index.first : resp + end + + def list_search_indexes() + resp = [] + backend do |b| + resp = b.get_search_index(nil) + end + resp.index ? resp.index : resp + end + + def delete_search_index(name) + raise ArgumentError, t("zero_length_index") if name.nil? || name.empty? + backend do |b| + b.delete_search_index(name) + end + true + end + + def create_search_schema(name, content) + raise ArgumentError, t("zero_length_schema") if name.nil? || name.empty? + raise ArgumentError, t("zero_length_content") if content.nil? || content.empty? + backend do |b| + b.create_search_schema(name, content) + end + true + end + + def get_search_schema(name) + raise ArgumentError, t("zero_length_schema") if name.nil? || name.empty? + backend do |b| + return b.get_search_schema(name) + end + end + end +end \ No newline at end of file diff --git a/lib/riak/locale/en.yml b/lib/riak/locale/en.yml index d439f24f..c77f48a9 100644 --- a/lib/riak/locale/en.yml +++ b/lib/riak/locale/en.yml @@ -66,3 +66,6 @@ en: wrong_argument_count_walk_spec: "wrong number of arguments (one Hash or bucket,tag,keep required)" zero_length_bucket: "bucket name cannot be a String of zero length" zero_length_key: "key cannot be a String of zero length" + zero_length_index: "index name cannot be a String of zero length" + zero_length_schema: "schema name cannot be a String of zero length" + zero_length_content: "content cannot be a String of zero length" diff --git a/spec/integration/yokozuna/index_spec.rb b/spec/integration/yokozuna/index_spec.rb new file mode 100644 index 00000000..7704fb43 --- /dev/null +++ b/spec/integration/yokozuna/index_spec.rb @@ -0,0 +1,74 @@ +require 'spec_helper' +require 'riak' + +describe "Yokozuna", test_server: true, integration: true do + before(:all) do + opts = { + http_port: test_server.http_port, + pb_port: test_server.pb_port, + protocol: 'pbc' + } + test_server.start + @client = Riak::Client.new opts + + @index = "test" + @schema = "testschema" + end + + context "without any indexes" do + it "should allow index creation" do + @client.create_search_index(@index).should == true + end + end + + context "with an index" do + before :all do + @client.create_search_index(@index).should == true + wait_until{ !@client.get_search_index(@index).nil? } + end + + it "should allow index inspection" do + @client.get_search_index(@index).name.should == @index + lambda{ @client.get_search_index("herp_derp") }.should raise_error(Riak::ProtobuffsFailedRequest) + end + + it "should have an index list" do + @client.list_search_indexes.size.should >= 1 + end + + it "should associate a bucket with an index" do + @bucket = Riak::Bucket.new(@client, @index) + @bucket.props = {'yz_index' => @index} + @bucket = @client.bucket(@index) + @bucket.props.should include('yz_index' => @index) + end + + context "associated with a bucket" do + before :all do + @bucket = Riak::Bucket.new(@client, @index) + @bucket.props = {'yz_index' => @index} + @bucket = @client.bucket(@index) + @bucket.props.should include('yz_index' => @index) + end + + it "should index on object writes" do + object = @bucket.get_or_new("cat") + object.raw_data = {"cat_s"=>"Lela"}.to_json + object.content_type = 'application/json' + object.store + sleep 1.1 # pause for index commit to trigger + + resp = @client.search(@index, "cat_s:Lela") + resp.should include('docs') + resp['docs'].size.should == 1 + end + end + end + + def wait_until(attempts=5) + begin + break if yield rescue nil + sleep 1 + end while (attempts -= 1) > 0 + end +end diff --git a/spec/integration/yokozuna/queries_spec.rb b/spec/integration/yokozuna/queries_spec.rb new file mode 100644 index 00000000..4c128846 --- /dev/null +++ b/spec/integration/yokozuna/queries_spec.rb @@ -0,0 +1,135 @@ +# encoding: UTF-8 +require 'spec_helper' +require 'riak' + +describe "Yokozona queries", test_server: true, integration: true do + before :all do + opts = { + http_port: test_server.http_port, + pb_port: test_server.pb_port, + protocol: 'pbc' + } + test_server.start + @client = Riak::Client.new opts + end + + context "with a schema and indexes" do + before :all do + @index = "test" + + @client.create_search_index(@index).should == true + wait_until{ !@client.get_search_index(@index).nil? } + @bucket = Riak::Bucket.new(@client, @index) + @bucket.props = {'yz_index' => @index} + + @o1 = build_json_obj(@bucket, "cat", {"cat_s"=>"Lela"}) + @o2 = build_json_obj(@bucket, "docs", {"dog_ss"=>["Einstein", "Olive"]}) + build_json_obj(@bucket, "Z", {"username_s"=>"Z", "name_s"=>"ryan", "age_i"=>30}) + build_json_obj(@bucket, "R", {"username_s"=>"R", "name_s"=>"eric", "age_i"=>34}) + build_json_obj(@bucket, "F", {"username_s"=>"F", "name_s"=>"bryan fink", "age_i"=>32}) + build_json_obj(@bucket, "H", {"username_s"=>"H", "name_s"=>"brett", "age_i"=>14}) + + sleep 1.1 # pause for index commit to trigger + end + + it "should produce results on single term queries" do + resp = @client.search(@index, "username_s:Z") + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should produce results on multiple term queries" do + resp = @client.search(@index, "username_s:(F OR H)") + resp.should include('docs') + resp['docs'].size.should == 2 + end + + it "should produce results on queries with boolean logic" do + resp = @client.search(@index, "username_s:Z AND name_s:ryan") + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should produce results on range queries" do + resp = @client.search(@index, "age_i:[30 TO 33]") + resp.should include('docs') + resp['docs'].size.should == 2 + end + + it "should produce results on phrase queries" do + resp = @client.search(@index, 'name_s:"bryan fink"') + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should produce results on wildcard queries" do + resp = @client.search(@index, "name_s:*ryan*") + resp.should include('docs') + resp['docs'].size.should == 2 + end + + it "should produce results on regexp queries" do + resp = @client.search(@index, "name_s:/br.*/") + resp.should include('docs') + resp['docs'].size.should == 2 + end + + # TODO: run this when pb utf8 works + it "should support utf8" do + build_json_obj(@bucket, "ja", {"text_ja"=>"私はハイビスカスを食べるのが 大好き"}) + # sleep 1.1 # pause for index commit to trigger + # resp = @client.search(@index, "text_ja:大好き") + # resp.should include('docs') + # resp['docs'].size.should == 1 + end + + context "using parameters" do + it "should search one row" do + resp = @client.search(@index, "*:*", {:rows => 1}) + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should search with df" do + resp = @client.search(@index, "Olive", {:rows => 1, :df => 'dog_ss'}) + resp.should include('docs') + resp['docs'].size.should == 1 + resp['docs'].first['dog_ss'] + end + + it "should produce top result on sort" do + resp = @client.search(@index, "username_s:*", {:sort => "age_i asc"}) + resp.should include('docs') + resp['docs'].first['age_i'].to_i.should == 14 + end + + end + + after(:all) do + # Can't delete index with associate buckets + lambda{ @client.delete_search_index(@index) }.should raise_error(Riak::ProtobuffsFailedRequest) + + # disassociate + @bucket = @client.bucket(@index) + @bucket.props = {'yz_index' => nil} + + @client.delete_search_index(@index) + end + end + + def wait_until(attempts=5) + begin + break if yield rescue nil + sleep 1 + end while (attempts -= 1) > 0 + end + + # populate objects + def build_json_obj(bucket, key, data) + object = bucket.get_or_new(key) + object.raw_data = data.to_json + object.content_type = 'application/json' + object.store + object + end +end diff --git a/spec/integration/yokozuna/schema_spec.rb b/spec/integration/yokozuna/schema_spec.rb new file mode 100644 index 00000000..56b6230b --- /dev/null +++ b/spec/integration/yokozuna/schema_spec.rb @@ -0,0 +1,61 @@ +require 'spec_helper' +require 'riak' + +describe "Yokozuna", test_server: true, integration: true do + before(:all) do + opts = { + http_port: test_server.http_port, + pb_port: test_server.pb_port, + protocol: 'pbc' + } + test_server.start + @client = Riak::Client.new opts + + @index = "test" + @schema = "testschema" + end + + context 'with no schema' do + it 'should allow schema creation' do + @client.create_search_schema(@schema, SCHEMA_CONTENT) + wait_until{ !@client.get_search_schema(@schema).nil? } + @client.get_search_schema(@schema).should_not be_nil + end + end + context 'with a schema' do + it 'should have a readable schema' do + @client.create_search_schema(@schema, SCHEMA_CONTENT) + wait_until{ !@client.get_search_schema(@schema).nil? } + schema_resp = @client.get_search_schema(@schema) + schema_resp.name.should == @schema + schema_resp.content.should == SCHEMA_CONTENT + end + end + + SCHEMA_CONTENT = <<-XML + + + + + + + + + + + + +_yz_id + + + + + XML + + def wait_until(attempts=5) + begin + break if yield rescue nil + sleep 1 + end while (attempts -= 1) > 0 + end +end