From 11a5b292982b811f838e64198bf1bbbd1b7dc433 Mon Sep 17 00:00:00 2001 From: Bryce Kerley Date: Tue, 27 Aug 2013 12:04:52 -0400 Subject: [PATCH 01/10] start yz integration specs --- spec/integration/yokozuna/index_spec.rb | 16 ++++++++++++++++ spec/integration/yokozuna/queries_spec.rb | 14 ++++++++++++++ spec/integration/yokozuna/schema_spec.rb | 8 ++++++++ 3 files changed, 38 insertions(+) create mode 100644 spec/integration/yokozuna/index_spec.rb create mode 100644 spec/integration/yokozuna/queries_spec.rb create mode 100644 spec/integration/yokozuna/schema_spec.rb diff --git a/spec/integration/yokozuna/index_spec.rb b/spec/integration/yokozuna/index_spec.rb new file mode 100644 index 00000000..344067ed --- /dev/null +++ b/spec/integration/yokozuna/index_spec.rb @@ -0,0 +1,16 @@ +describe "Yokozuna" do + context "without any indexes" do + it "should allow index creation" + end + + context "with an index" do + it "should have an index list" + it "should allow index inspection" + + it "should associate an index with a bucket" + + context "associated with a bucket" do + it "should index on object writes" + end + end +end diff --git a/spec/integration/yokozuna/queries_spec.rb b/spec/integration/yokozuna/queries_spec.rb new file mode 100644 index 00000000..afe4c8da --- /dev/null +++ b/spec/integration/yokozuna/queries_spec.rb @@ -0,0 +1,14 @@ +describe "Yokozona queries with a schema and indexes" do + it "should produce results on single term queries" + it "should produce results on multiple term queries" + it "should produce results on queries with boolean logic" + it "should produce results on range queries" + it "should produce results on phrase queries" + it "should produce results on wildcard queries" + it "should produce results on regexp queries" + + context "using parameters" do + it "should support start/offset parameters" + it "should support rows/limit parameters" + end +end diff --git a/spec/integration/yokozuna/schema_spec.rb b/spec/integration/yokozuna/schema_spec.rb new file mode 100644 index 00000000..47ffe1da --- /dev/null +++ b/spec/integration/yokozuna/schema_spec.rb @@ -0,0 +1,8 @@ +describe 'Yokozuna' do + context 'with no schema' do + it 'should allow schema creation' + end + context 'with a schema' do + it 'should have a readable schema' + end +end From ac1140f6b8064e8da6891fc7dea40177309d18d9 Mon Sep 17 00:00:00 2001 From: Bryce Kerley Date: Tue, 27 Aug 2013 17:55:39 -0400 Subject: [PATCH 02/10] Use beefcake to generate the list of messages from riak_pb --- .gitignore | 1 + Rakefile | 26 +++++++++++++ lib/riak/client/beefcake/footer | 4 ++ lib/riak/client/beefcake/header | 6 +++ lib/riak/client/beefcake/message_overlay.rb | 39 +++++++++++++++++++ .../client/beefcake_protobuffs_backend.rb | 1 + 6 files changed, 77 insertions(+) create mode 100644 lib/riak/client/beefcake/footer create mode 100644 lib/riak/client/beefcake/header create mode 100644 lib/riak/client/beefcake/message_overlay.rb diff --git a/.gitignore b/.gitignore index 81a9f638..6fa90851 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ Gemfile.lock vendor/erlang vendor/riak .ruby-version +tmp diff --git a/Rakefile b/Rakefile index ed36a8a9..2e67245d 100644 --- a/Rakefile +++ b/Rakefile @@ -70,3 +70,29 @@ RSpec::Core::RakeTask.new(:ci) do |spec| spec.rspec_opts = %w[--profile] end task :default => :ci + + +desc "Generate Protocol Buffers message definitions from riak_pb" +task :pb_defs => 'beefcake:pb_defs' + +namespace :beefcake do + task :pb_defs => 'lib/riak/client/beefcake/messages.rb' + + file 'lib/riak/client/beefcake/messages.rb' => %w{tmp/riak_kv.pb.rb tmp/riak_yokozuna.pb.rb} do |t| + sh "cat lib/riak/client/beefcake/header tmp/riak.pb.rb #{t.prerequisites.join ' '} lib/riak/client/beefcake/footer > #{t.name}" + end + + file 'tmp/riak_kv.pb.rb' => 'tmp/riak_pb' do |t| + sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/riak_kv.proto" + end + + file 'tmp/riak_yokozuna.pb.rb' => 'tmp/riak_pb' do |t| + sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/riak_yokozuna.proto" + end + + directory 'tmp/riak_pb' do + cd 'tmp' do + sh "git clone -b develop https://github.com/basho/riak_pb.git" + end + end +end diff --git a/lib/riak/client/beefcake/footer b/lib/riak/client/beefcake/footer new file mode 100644 index 00000000..f8fdd785 --- /dev/null +++ b/lib/riak/client/beefcake/footer @@ -0,0 +1,4 @@ + + end + end +end diff --git a/lib/riak/client/beefcake/header b/lib/riak/client/beefcake/header new file mode 100644 index 00000000..960f1bc1 --- /dev/null +++ b/lib/riak/client/beefcake/header @@ -0,0 +1,6 @@ +require 'beefcake' + +module Riak + class Client + # @private + class BeefcakeProtobuffsBackend diff --git a/lib/riak/client/beefcake/message_overlay.rb b/lib/riak/client/beefcake/message_overlay.rb new file mode 100644 index 00000000..378983f0 --- /dev/null +++ b/lib/riak/client/beefcake/message_overlay.rb @@ -0,0 +1,39 @@ +module Riak + class Client + # @private + class BeefcakeProtobuffsBackend + class RpbIndexReq + module IndexQueryType + EQ = 0 + RANGE = 1 + end + end + + class RpbBucketProps + + # "repeated" elements with zero items are indistinguishable + # from a nil, so we have to manage has_precommit/has_postcommit + # flags. + def precommit=(newval) + @precommit = newval + @has_precommit = !!newval + end + + def has_precommit=(newval) + @has_precommit = newval + @precommit ||= [] if newval + end + + def postcommit=(newval) + @postcommit = newval + @has_postcommit = !!newval + end + + def has_postcommit=(newval) + @has_postcommit = newval + @postcommit ||= [] if newval + end + end + end + end +end diff --git a/lib/riak/client/beefcake_protobuffs_backend.rb b/lib/riak/client/beefcake_protobuffs_backend.rb index 94ed84b8..17f1f8fc 100644 --- a/lib/riak/client/beefcake_protobuffs_backend.rb +++ b/lib/riak/client/beefcake_protobuffs_backend.rb @@ -11,6 +11,7 @@ def self.configured? begin require 'beefcake' require 'riak/client/beefcake/messages' + require 'riak/client/beefcake/message_overlay' require 'riak/client/beefcake/object_methods' true rescue LoadError, NameError From 1d780d76d8c3c38c4da8d7223942faae628368df Mon Sep 17 00:00:00 2001 From: Bryce Kerley Date: Tue, 27 Aug 2013 17:55:52 -0400 Subject: [PATCH 03/10] Generated message list --- lib/riak/client/beefcake/messages.rb | 908 ++++++++++++++++----------- 1 file changed, 532 insertions(+), 376 deletions(-) diff --git a/lib/riak/client/beefcake/messages.rb b/lib/riak/client/beefcake/messages.rb index e85936fd..99e96e83 100644 --- a/lib/riak/client/beefcake/messages.rb +++ b/lib/riak/client/beefcake/messages.rb @@ -4,382 +4,538 @@ module Riak class Client # @private class BeefcakeProtobuffsBackend - # Embedded messages - class RpbPair - include Beefcake::Message - required :key, :bytes, 1 - optional :value, :bytes, 2 - end - - # module-function pair for commit hooks and other properties that take - # functions - class RpbModFun - include Beefcake::Message - - required :module, :bytes, 1 - required :function, :bytes, 2 - end - - class RpbCommitHook - include Beefcake::Message - - optional :modfun, RpbModFun, 1 - optional :name, :bytes, 2 - end - - class RpbBucketProps - include Beefcake::Message - - # riak_core_app - optional :n_val, :uint32, 1 - optional :allow_mult, :bool, 2 - optional :last_write_wins, :bool, 3 - - # riak_core values with special handling, see below - repeated :precommit, RpbCommitHook, 4 - optional :has_precommit, :bool, 5, :default => false - repeated :postcommit, RpbCommitHook, 6 - optional :has_postcommit, :bool, 7, :default => false - - optional :chash_keyfun, RpbModFun, 8 - - # riak_kv_app - optional :linkfun, RpbModFun, 9 - optional :old_vclock, :uint32, 10 - optional :young_vclock, :uint32, 11 - optional :big_vclock, :uint32, 12 - optional :small_vclock, :uint32, 13 - optional :pr, :uint32, 14 - optional :r, :uint32, 15 - optional :w, :uint32, 16 - optional :pw, :uint32, 17 - optional :dw, :uint32, 18 - optional :rw, :uint32, 19 - optional :basic_quorum, :bool, 20 - optional :notfound_ok, :bool, 21 - - # riak_kv_multi_backend - optional :backend, :bytes, 22 - - # riak_search bucket fixup - optional :search, :bool, 23 - - module RpbReplMode - FALSE = 0 - REALTIME = 1 - FULLSYNC = 2 - TRUE = 3 - end - - optional :repl, RpbReplMode, 24 - - # "repeated" elements with zero items are indistinguishable - # from a nil, so we have to manage has_precommit/has_postcommit - # flags. - def precommit=(newval) - @precommit = newval - @has_precommit = !!newval - end - - def has_precommit=(newval) - @has_precommit = newval - @precommit ||= [] if newval - end - - def postcommit=(newval) - @postcommit = newval - @has_postcommit = !!newval - end - - def has_postcommit=(newval) - @has_postcommit = newval - @postcommit ||= [] if newval - end - end - - class RpbLink - include Beefcake::Message - optional :bucket, :bytes, 1 - optional :key, :bytes, 2 - optional :tag, :bytes, 3 - end - - class RpbContent - include Beefcake::Message - required :value, :bytes, 1 - optional :content_type, :bytes, 2 - optional :charset, :bytes, 3 - optional :content_encoding, :bytes, 4 - optional :vtag, :bytes, 5 - repeated :links, RpbLink, 6 - optional :last_mod, :uint32, 7 - optional :last_mod_usecs, :uint32, 8 - repeated :usermeta, RpbPair, 9 - repeated :indexes, RpbPair, 10 - end - - # Primary messages - class RpbErrorResp - include Beefcake::Message - required :errmsg, :bytes, 1 - required :errcode, :uint32, 2 - end - - class RpbGetClientIdResp - include Beefcake::Message - required :client_id, :bytes, 1 - end - - class RpbSetClientIdReq - include Beefcake::Message - required :client_id, :bytes, 1 - end - - class RpbGetServerInfoResp - include Beefcake::Message - optional :node, :bytes, 1 - optional :server_version, :bytes, 2 - end - - class RpbGetReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :key, :bytes, 2 - optional :r, :uint32, 3 - optional :pr, :uint32, 4 - optional :basic_quorum, :bool, 5 - optional :notfound_ok, :bool, 6 - optional :if_modified, :bytes, 7 - optional :head, :bool, 8 - optional :deletedvclock, :bool, 9 - optional :timeout, :uint32, 10 - optional :sloppy_quorum, :bool, 11 - optional :n_val, :uint32, 12 - end - - class RpbGetResp - include Beefcake::Message - repeated :content, RpbContent, 1 - optional :vclock, :bytes, 2 - optional :unchanged, :bool, 3 - end - - class RpbPutReq - include Beefcake::Message - required :bucket, :bytes, 1 - optional :key, :bytes, 2 - optional :vclock, :bytes, 3 - required :content, RpbContent, 4 - optional :w, :uint32, 5 - optional :dw, :uint32, 6 - optional :returnbody, :bool, 7 - optional :pw, :uint32, 8 - optional :if_not_modified, :bool, 9 - optional :if_none_match, :bool, 10 - optional :return_head, :bool, 11 - optional :timeout, :uint32, 12 - optional :asis, :bool, 13 - optional :sloppy_quorum, :bool, 14 - optional :n_val, :uint32, 15 - end - - class RpbPutResp - include Beefcake::Message - repeated :content, RpbContent, 1 - optional :vclock, :bytes, 2 - optional :key, :bytes, 3 - end - - class RpbDelReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :key, :bytes, 2 - optional :rw, :uint32, 3 - optional :vclock, :bytes, 4 - optional :r, :uint32, 5 - optional :w, :uint32, 6 - optional :pr, :uint32, 7 - optional :pw, :uint32, 8 - optional :dw, :uint32, 9 - optional :timeout, :uint32, 10 - optional :sloppy_quorum, :bool, 11 - optional :n_val, :uint32, 12 - end - - class RpbListBucketsReq - include Beefcake::Message - optional :timeout, :uint32, 1 - optional :stream, :bool, 2 - end - - class RpbListBucketsResp - include Beefcake::Message - repeated :buckets, :bytes, 1 - optional :done, :bool, 2 - end - - class RpbListKeysReq - include Beefcake::Message - required :bucket, :bytes, 1 - optional :timeout, :uint32, 2 - end - - class RpbListKeysResp - include Beefcake::Message - repeated :keys, :bytes, 1 - optional :done, :bool, 2 - end - - class RpbGetBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - end - - class RpbGetBucketResp - include Beefcake::Message - required :props, RpbBucketProps, 1 - end - - class RpbSetBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :props, RpbBucketProps, 2 - end - - class RpbResetBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - end - - class RpbMapRedReq - include Beefcake::Message - required :request, :bytes, 1 - required :content_type, :bytes, 2 - end - - class RpbMapRedResp - include Beefcake::Message - optional :phase, :uint32, 1 - optional :response, :bytes, 2 - optional :done, :bool, 3 - end - - class RpbIndexReq - include Beefcake::Message - module IndexQueryType - EQ = 0 - RANGE = 1 - end - - required :bucket, :bytes, 1 - required :index, :bytes, 2 - required :qtype, IndexQueryType, 3 - optional :key, :bytes, 4 - optional :range_min, :bytes, 5 - optional :range_max, :bytes, 6 - optional :return_terms, :bool, 7 - optional :stream, :bool, 8 - optional :max_results, :uint32, 9 - optional :continuation, :bytes, 10 - optional :timeout, :uint32, 11 - end - - class RpbIndexResp - include Beefcake::Message - repeated :keys, :bytes, 1 - repeated :results, RpbPair, 2 - optional :continuation, :bytes, 3 - optional :done, :bool, 4 - end - - class RpbSearchDoc - include Beefcake::Message - # We have to name this differently than the .proto file does - # because Beefcake uses 'fields' as an instance method. - repeated :properties, RpbPair, 1 - end - - class RpbSearchQueryReq - include Beefcake::Message - required :q, :bytes, 1 - required :index, :bytes, 2 - optional :rows, :uint32, 3 - optional :start, :uint32, 4 - optional :sort, :bytes, 5 - optional :filter, :bytes, 6 - optional :df, :bytes, 7 - optional :op, :bytes, 8 - repeated :fl, :bytes, 9 - optional :presort, :bytes, 10 - end - - class RpbSearchQueryResp - include Beefcake::Message - repeated :docs, RpbSearchDoc, 1, :default => [] - optional :max_score, :float, 2 - optional :num_found, :uint32, 3 - end - - class RpbResetBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - end - - class RpbCSBucketReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :start_key, :bytes, 2 - optional :end_key, :bytes, 3 - optional :start_incl, :bool, 4, default: true - optional :end_incl, :bool, 5, default: false - optional :continuation, :bytes, 6 - optional :max_results, :uint32, 7 - optional :timeout, :uint32, 8 - end - - class RpbIndexObject - include Beefcake::Message - required :key, :bytes, 1 - required :object, RpbGetResp, 2 - end - - class RpbCSBucketResp - include Beefcake::Message - repeated :objects, RpbIndexObject, 1 - optional :continuation, :bytes, 2 - optional :done, :bool, 3 - end - - class RpbCounterUpdateReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :key, :bytes, 2 - required :amount, :sint64, 3 - optional :w, :uint32, 4 - optional :dw, :uint32, 5 - optional :pw, :uint32, 6 - optional :returnvalue, :bool, 7 - end - - class RpbCounterUpdateResp - include Beefcake::Message - optional :value, :sint64, 1 - end - - class RpbCounterGetReq - include Beefcake::Message - required :bucket, :bytes, 1 - required :key, :bytes, 2 - optional :r, :uint32, 3 - optional :pr, :uint32, 4 - optional :basic_quorum, :bool, 5 - optional :notfound_ok, :bool, 6 - end - - class RpbCounterGetResp - include Beefcake::Message - optional :value, :sint64, 1 - end +## Generated from riak.proto for +require "beefcake" + + +class RpbErrorResp + include Beefcake::Message +end + + +class RpbGetServerInfoResp + include Beefcake::Message +end + + +class RpbPair + include Beefcake::Message +end + + +class RpbGetBucketReq + include Beefcake::Message +end + + +class RpbGetBucketResp + include Beefcake::Message +end + + +class RpbSetBucketReq + include Beefcake::Message +end + + +class RpbResetBucketReq + include Beefcake::Message +end + + +class RpbModFun + include Beefcake::Message +end + + +class RpbCommitHook + include Beefcake::Message +end + + +class RpbBucketProps + include Beefcake::Message +end + + +class RpbErrorResp + required :errmsg, :bytes, 1 + required :errcode, :uint32, 2 +end + +class RpbGetServerInfoResp + optional :node, :bytes, 1 + optional :server_version, :bytes, 2 +end + +class RpbPair + required :key, :bytes, 1 + optional :value, :bytes, 2 +end + +class RpbGetBucketReq + required :bucket, :bytes, 1 +end + +class RpbGetBucketResp + required :props, RpbBucketProps, 1 +end + +class RpbSetBucketReq + required :bucket, :bytes, 1 + required :props, RpbBucketProps, 2 +end + +class RpbResetBucketReq + required :bucket, :bytes, 1 +end + +class RpbModFun + required :module, :bytes, 1 + required :function, :bytes, 2 +end + +class RpbCommitHook + optional :modfun, RpbModFun, 1 + optional :name, :bytes, 2 +end + +class RpbBucketProps + module RpbReplMode + FALSE = 0 + REALTIME = 1 + FULLSYNC = 2 + TRUE = 3 + end + optional :n_val, :uint32, 1 + optional :allow_mult, :bool, 2 + optional :last_write_wins, :bool, 3 + repeated :precommit, RpbCommitHook, 4 + optional :has_precommit, :bool, 5, :default => false + repeated :postcommit, RpbCommitHook, 6 + optional :has_postcommit, :bool, 7, :default => false + optional :chash_keyfun, RpbModFun, 8 + optional :linkfun, RpbModFun, 9 + optional :old_vclock, :uint32, 10 + optional :young_vclock, :uint32, 11 + optional :big_vclock, :uint32, 12 + optional :small_vclock, :uint32, 13 + optional :pr, :uint32, 14 + optional :r, :uint32, 15 + optional :w, :uint32, 16 + optional :pw, :uint32, 17 + optional :dw, :uint32, 18 + optional :rw, :uint32, 19 + optional :basic_quorum, :bool, 20 + optional :notfound_ok, :bool, 21 + optional :backend, :bytes, 22 + optional :search, :bool, 23 + optional :repl, RpbBucketProps::RpbReplMode, 24 + optional :yz_index, :bytes, 25 +end +## Generated from riak_kv.proto for +require "beefcake" + + +class RpbGetClientIdResp + include Beefcake::Message +end + + +class RpbSetClientIdReq + include Beefcake::Message +end + + +class RpbGetReq + include Beefcake::Message +end + + +class RpbGetResp + include Beefcake::Message +end + + +class RpbPutReq + include Beefcake::Message +end + + +class RpbPutResp + include Beefcake::Message +end + + +class RpbDelReq + include Beefcake::Message +end + + +class RpbListBucketsReq + include Beefcake::Message +end + + +class RpbListBucketsResp + include Beefcake::Message +end + + +class RpbListKeysReq + include Beefcake::Message +end + + +class RpbListKeysResp + include Beefcake::Message +end + + +class RpbMapRedReq + include Beefcake::Message +end + + +class RpbMapRedResp + include Beefcake::Message +end + + +class RpbIndexReq + include Beefcake::Message +end + + +class RpbIndexResp + include Beefcake::Message +end + + +class RpbCSBucketReq + include Beefcake::Message +end + + +class RpbCSBucketResp + include Beefcake::Message +end + + +class RpbIndexObject + include Beefcake::Message +end + + +class RpbContent + include Beefcake::Message +end + + +class RpbLink + include Beefcake::Message +end + + +class RpbCounterUpdateReq + include Beefcake::Message +end + + +class RpbCounterUpdateResp + include Beefcake::Message +end + + +class RpbCounterGetReq + include Beefcake::Message +end + + +class RpbCounterGetResp + include Beefcake::Message +end + + +class RpbGetClientIdResp + required :client_id, :bytes, 1 +end + +class RpbSetClientIdReq + required :client_id, :bytes, 1 +end + +class RpbGetReq + required :bucket, :bytes, 1 + required :key, :bytes, 2 + optional :r, :uint32, 3 + optional :pr, :uint32, 4 + optional :basic_quorum, :bool, 5 + optional :notfound_ok, :bool, 6 + optional :if_modified, :bytes, 7 + optional :head, :bool, 8 + optional :deletedvclock, :bool, 9 + optional :timeout, :uint32, 10 + optional :sloppy_quorum, :bool, 11 + optional :n_val, :uint32, 12 +end + +class RpbGetResp + repeated :content, RpbContent, 1 + optional :vclock, :bytes, 2 + optional :unchanged, :bool, 3 +end + +class RpbPutReq + required :bucket, :bytes, 1 + optional :key, :bytes, 2 + optional :vclock, :bytes, 3 + required :content, RpbContent, 4 + optional :w, :uint32, 5 + optional :dw, :uint32, 6 + optional :return_body, :bool, 7 + optional :pw, :uint32, 8 + optional :if_not_modified, :bool, 9 + optional :if_none_match, :bool, 10 + optional :return_head, :bool, 11 + optional :timeout, :uint32, 12 + optional :asis, :bool, 13 + optional :sloppy_quorum, :bool, 14 + optional :n_val, :uint32, 15 +end + +class RpbPutResp + repeated :content, RpbContent, 1 + optional :vclock, :bytes, 2 + optional :key, :bytes, 3 +end + +class RpbDelReq + required :bucket, :bytes, 1 + required :key, :bytes, 2 + optional :rw, :uint32, 3 + optional :vclock, :bytes, 4 + optional :r, :uint32, 5 + optional :w, :uint32, 6 + optional :pr, :uint32, 7 + optional :pw, :uint32, 8 + optional :dw, :uint32, 9 + optional :timeout, :uint32, 10 + optional :sloppy_quorum, :bool, 11 + optional :n_val, :uint32, 12 +end + +class RpbListBucketsReq + optional :timeout, :uint32, 1 + optional :stream, :bool, 2 +end + +class RpbListBucketsResp + repeated :buckets, :bytes, 1 + optional :done, :bool, 2 +end + +class RpbListKeysReq + required :bucket, :bytes, 1 + optional :timeout, :uint32, 2 +end + +class RpbListKeysResp + repeated :keys, :bytes, 1 + optional :done, :bool, 2 +end + +class RpbMapRedReq + required :request, :bytes, 1 + required :content_type, :bytes, 2 +end + +class RpbMapRedResp + optional :phase, :uint32, 1 + optional :response, :bytes, 2 + optional :done, :bool, 3 +end + +class RpbIndexReq + module IndexQueryType + eq = 0 + range = 1 + end + required :bucket, :bytes, 1 + required :index, :bytes, 2 + required :qtype, RpbIndexReq::IndexQueryType, 3 + optional :key, :bytes, 4 + optional :range_min, :bytes, 5 + optional :range_max, :bytes, 6 + optional :return_terms, :bool, 7 + optional :stream, :bool, 8 + optional :max_results, :uint32, 9 + optional :continuation, :bytes, 10 + optional :timeout, :uint32, 11 +end + +class RpbIndexResp + repeated :keys, :bytes, 1 + repeated :results, RpbPair, 2 + optional :continuation, :bytes, 3 + optional :done, :bool, 4 +end + +class RpbCSBucketReq + required :bucket, :bytes, 1 + required :start_key, :bytes, 2 + optional :end_key, :bytes, 3 + optional :start_incl, :bool, 4, :default => true + optional :end_incl, :bool, 5, :default => false + optional :continuation, :bytes, 6 + optional :max_results, :uint32, 7 + optional :timeout, :uint32, 8 +end + +class RpbCSBucketResp + repeated :objects, RpbIndexObject, 1 + optional :continuation, :bytes, 2 + optional :done, :bool, 3 +end + +class RpbIndexObject + required :key, :bytes, 1 + required :object, RpbGetResp, 2 +end + +class RpbContent + required :value, :bytes, 1 + optional :content_type, :bytes, 2 + optional :charset, :bytes, 3 + optional :content_encoding, :bytes, 4 + optional :vtag, :bytes, 5 + repeated :links, RpbLink, 6 + optional :last_mod, :uint32, 7 + optional :last_mod_usecs, :uint32, 8 + repeated :usermeta, RpbPair, 9 + repeated :indexes, RpbPair, 10 + optional :deleted, :bool, 11 +end + +class RpbLink + optional :bucket, :bytes, 1 + optional :key, :bytes, 2 + optional :tag, :bytes, 3 +end + +class RpbCounterUpdateReq + required :bucket, :bytes, 1 + required :key, :bytes, 2 + required :amount, :sint64, 3 + optional :w, :uint32, 4 + optional :dw, :uint32, 5 + optional :pw, :uint32, 6 + optional :returnvalue, :bool, 7 +end + +class RpbCounterUpdateResp + optional :value, :sint64, 1 +end + +class RpbCounterGetReq + required :bucket, :bytes, 1 + required :key, :bytes, 2 + optional :r, :uint32, 3 + optional :pr, :uint32, 4 + optional :basic_quorum, :bool, 5 + optional :notfound_ok, :bool, 6 +end + +class RpbCounterGetResp + optional :value, :sint64, 1 +end +## Generated from riak_yokozuna.proto for +require "beefcake" + + +class RpbYokozunaIndex + include Beefcake::Message +end + + +class RpbYokozunaIndexGetReq + include Beefcake::Message +end + + +class RpbYokozunaIndexGetResp + include Beefcake::Message +end + + +class RpbYokozunaIndexPutReq + include Beefcake::Message +end + + +class RpbYokozunaIndexDeleteReq + include Beefcake::Message +end + + +class RpbYokozunaSchema + include Beefcake::Message +end + + +class RpbYokozunaSchemaPutReq + include Beefcake::Message +end + + +class RpbYokozunaSchemaGetReq + include Beefcake::Message +end + + +class RpbYokozunaSchemaGetResp + include Beefcake::Message +end + + +class RpbYokozunaIndex + required :name, :bytes, 1 + optional :schema, :bytes, 2 +end + +class RpbYokozunaIndexGetReq + optional :name, :bytes, 1 +end + +class RpbYokozunaIndexGetResp + repeated :index, RpbYokozunaIndex, 1 +end + +class RpbYokozunaIndexPutReq + required :index, RpbYokozunaIndex, 1 +end + +class RpbYokozunaIndexDeleteReq + required :name, :bytes, 1 +end + +class RpbYokozunaSchema + required :name, :bytes, 1 + optional :content, :bytes, 2 +end + +class RpbYokozunaSchemaPutReq + required :schema, RpbYokozunaSchema, 1 +end + +class RpbYokozunaSchemaGetReq + required :name, :bytes, 1 +end + +class RpbYokozunaSchemaGetResp + required :schema, RpbYokozunaSchema, 1 +end + end end end From 18d81da17d33e671448fcf21a34ff3b048bdc982 Mon Sep 17 00:00:00 2001 From: Bryce Kerley Date: Tue, 27 Aug 2013 18:26:02 -0400 Subject: [PATCH 04/10] Add yz message codes --- lib/riak/client/beefcake/message_codes.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/riak/client/beefcake/message_codes.rb b/lib/riak/client/beefcake/message_codes.rb index 597f1275..028f41b7 100644 --- a/lib/riak/client/beefcake/message_codes.rb +++ b/lib/riak/client/beefcake/message_codes.rb @@ -39,6 +39,13 @@ module BeefcakeMessageCodes :CounterUpdateResp => 51, :CounterGetReq => 52, :CounterGetResp => 53, + :YokozunaIndexGetReq => 54, + :YokozunaIndexGetResp => 55, + :YokozunaIndexPutReq => 56, + :YokozunaIndexDeleteReq => 57, + :YokozunaSchemaGetReq => 58, + :YokozunaSchemaGetResp => 59, + :YokozunaSchemaPutReq => 60, } CODE_TO_MESSAGE = MESSAGE_TO_CODE.invert From f17c66d695550cd0d840d26327d4e732591faff3 Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Tue, 27 Aug 2013 18:29:29 -0700 Subject: [PATCH 05/10] Add admin defs to pb backend and client --- lib/riak/client.rb | 1 + lib/riak/client/beefcake/messages.rb | 2 +- .../client/beefcake_protobuffs_backend.rb | 46 ++++++- lib/riak/client/yokozuna.rb | 52 +++++++ lib/riak/locale/en.yml | 3 + spec/integration/riak/yokozuna_spec.rb | 128 ++++++++++++++++++ 6 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 lib/riak/client/yokozuna.rb create mode 100644 spec/integration/riak/yokozuna_spec.rb diff --git a/lib/riak/client.rb b/lib/riak/client.rb index 2092eff2..ee906368 100644 --- a/lib/riak/client.rb +++ b/lib/riak/client.rb @@ -8,6 +8,7 @@ require 'riak/client/decaying' require 'riak/client/node' require 'riak/client/search' +require 'riak/client/yokozuna' require 'riak/client/http_backend' require 'riak/client/net_http_backend' require 'riak/client/excon_backend' diff --git a/lib/riak/client/beefcake/messages.rb b/lib/riak/client/beefcake/messages.rb index 99e96e83..03c3116c 100644 --- a/lib/riak/client/beefcake/messages.rb +++ b/lib/riak/client/beefcake/messages.rb @@ -538,4 +538,4 @@ class RpbYokozunaSchemaGetResp end end -end +end \ No newline at end of file diff --git a/lib/riak/client/beefcake_protobuffs_backend.rb b/lib/riak/client/beefcake_protobuffs_backend.rb index 17f1f8fc..706bb08c 100644 --- a/lib/riak/client/beefcake_protobuffs_backend.rb +++ b/lib/riak/client/beefcake_protobuffs_backend.rb @@ -211,6 +211,44 @@ def search(index, query, options={}) decode_response end + def create_search_index(name, schema=nil) + index = RpbYokozunaIndex.new(:name => name, :schema => schema) + req = RpbYokozunaIndexPutReq.new(:index => index) + write_protobuff(:YokozunaIndexPutReq, req) + decode_response + end + + def get_search_index(name) + req = RpbYokozunaIndexGetReq.new(:name => name) + write_protobuff(:YokozunaIndexGetReq, req) + resp = decode_response + if resp.index && Array === resp + resp.index.map{|index| {:name => index.name, :schema => index.schema} } + else + resp + end + end + + def delete_search_index(name) + req = RpbYokozunaIndexDeleteReq.new(:name => name) + write_protobuff(:YokozunaIndexDeleteReq, req) + decode_response + end + + def create_search_schema(name, content) + schema = RpbYokozunaSchema.new(:name => name, :content => content) + req = RpbYokozunaSchemaPutReq.new(:schema => schema) + write_protobuff(:YokozunaSchemaPutReq, req) + decode_response + end + + def get_search_schema(name) + req = RpbYokozunaSchemaGetReq.new(:name => name) + write_protobuff(:YokozunaSchemaGetReq, req) + resp = decode_response + resp.schema ? resp.schema : resp + end + private def write_protobuff(code, message) encoded = message.encode @@ -235,7 +273,9 @@ def decode_response(*args) :ListKeysResp, :IndexResp [] - when :GetResp + when :GetResp, + :YokozunaIndexGetResp, + :YokozunaSchemaGetResp raise Riak::ProtobuffsFailedRequest.new(:not_found, t('not_found')) when :CounterGetResp, :CounterUpdateResp @@ -287,6 +327,10 @@ def decode_response(*args) when :CounterGetResp res = RpbCounterGetResp.decode message res.value || 0 + when :YokozunaIndexGetResp + res = RpbYokozunaIndexGetResp.decode message + when :YokozunaSchemaGetResp + res = RpbYokozunaSchemaGetResp.decode message end end rescue SystemCallError, SocketError => e diff --git a/lib/riak/client/yokozuna.rb b/lib/riak/client/yokozuna.rb new file mode 100644 index 00000000..206148e8 --- /dev/null +++ b/lib/riak/client/yokozuna.rb @@ -0,0 +1,52 @@ +module Riak + class Client + def create_search_index(name, schema=nil) + raise ArgumentError, t("zero_length_index") if name.nil? || name.empty? + backend do |b| + b.create_search_index(name, schema) + end + true + end + + def get_search_index(name) + raise ArgumentError, t("zero_length_index") if name.nil? || name.empty? + resp = [] + backend do |b| + resp = b.get_search_index(name) + end + resp.index && Array === resp.index ? resp.index.first : resp + end + + def list_search_indexes() + resp = [] + backend do |b| + resp = b.get_search_index(nil) + end + resp.index ? resp.index : resp + end + + def delete_search_index(name) + raise ArgumentError, t("zero_length_index") if name.nil? || name.empty? + backend do |b| + b.delete_search_index(name) + end + true + end + + def create_search_schema(name, content) + raise ArgumentError, t("zero_length_schema") if name.nil? || name.empty? + raise ArgumentError, t("zero_length_content") if content.nil? || content.empty? + backend do |b| + b.create_search_schema(name, content) + end + true + end + + def get_search_schema(name) + raise ArgumentError, t("zero_length_schema") if name.nil? || name.empty? + backend do |b| + return b.get_search_schema(name) + end + end + end +end \ No newline at end of file diff --git a/lib/riak/locale/en.yml b/lib/riak/locale/en.yml index d439f24f..c77f48a9 100644 --- a/lib/riak/locale/en.yml +++ b/lib/riak/locale/en.yml @@ -66,3 +66,6 @@ en: wrong_argument_count_walk_spec: "wrong number of arguments (one Hash or bucket,tag,keep required)" zero_length_bucket: "bucket name cannot be a String of zero length" zero_length_key: "key cannot be a String of zero length" + zero_length_index: "index name cannot be a String of zero length" + zero_length_schema: "schema name cannot be a String of zero length" + zero_length_content: "content cannot be a String of zero length" diff --git a/spec/integration/riak/yokozuna_spec.rb b/spec/integration/riak/yokozuna_spec.rb new file mode 100644 index 00000000..0ded77f5 --- /dev/null +++ b/spec/integration/riak/yokozuna_spec.rb @@ -0,0 +1,128 @@ +# require 'spec_helper' +$LOAD_PATH.unshift(File.dirname(__FILE__)) +$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', '..', '..', 'lib')) + +require 'rubygems' # Use the gems path only for the spec suite +require 'riak' +require 'rspec' + +RSpec.configure do |c| + # declare an exclusion filter + @http_port ||= 10018 # test_server.http_port + @client = Riak::Client.new(:http_port => @http_port) + yz_active = @client.ping rescue false + c.filter_run_excluding :yokozuna => !yz_active +end + +describe "Yokozuna", :yokozuna => true do + before(:all) do + @pbc_port ||= 10017 # test_server.pb_port + @http_port ||= 10018 # test_server.http_port + @client = Riak::Client.new(:http_port => @http_port, :pb_port => @pbc_port, :protocol => "pbc") + + @index = "test" + @schema = "testschema" + end + + context "using the admin client" do + it "should create / get / list indexes" do + @client.create_search_index(@index).should == true + # TODO: replace this with a wait function + sleep 1.1 + @client.get_search_index(@index).name.should == @index + @client.list_search_indexes.size.should >= 1 + lambda{ @client.get_search_index("herp_derp") }.should raise_error(Riak::ProtobuffsFailedRequest) + end + + it "should associate a bucket with an index" do + @bucket = Riak::Bucket.new(@client, @index) + @bucket.props = {'yz_index' => @index} + @bucket = @client.bucket(@index) + @bucket.props.should include('yz_index' => @index) + end + + it "should create / get schemas" do + content = <<-XML + + + + + + + + + + + + +_yz_id + + + + + XML + @client.create_search_schema(@schema, content) + schema_resp = @client.get_search_schema(@schema) + schema_resp.name.should == @schema + schema_resp.content.should == content + end + end + + context "using the search client" do + before(:all) do + @bucket = @client.bucket(@index) + + @object = @bucket.get_or_new("cat") + @object.raw_data = '{"cat_s":"Lela"}' + @object.content_type = 'application/json' + @object.store + + @object2 = @bucket.get_or_new("dogs") + @object2.raw_data = '{"dog_ss":["Einstein", "Olive"]}' + @object2.content_type = 'application/json' + @object2.store + + sleep 1.1 + end + + it "should search one row" do + resp = @client.search("test", "*:*", {:rows => 1}) + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should search with df" do + resp = @client.search("test", "Olive", {:rows => 1, :df => 'dog_ss'}) + resp.should include('docs') + resp['docs'].size.should == 1 + resp['docs'].first['dog_ss'] + end + end + + # context "using the HTTP driver" do + # it "should search as XML" do + # resp = @http_client.search("test", "*:*", {:rows => 1, :wt => 'xml'}) + # root = REXML::Document.new(resp).root + # root.name.should == 'response' + # doc = root.elements['result'].elements['doc'] + # doc.should_not be_nil + # cats = doc.elements["str[@name='cat_s']"] + # cats.should have(1).item + # cats.text.should == 'Lela' + # end + # it "should search as JSON with df" do + # resp = @http_client.search("test", "Olive", {:rows => 1, :wt => 'json', :df => 'dog_ss'}) + # end + # end + + after(:all) do + # Can't delete index with associate buckets + lambda{ @client.delete_search_index(@index) }.should raise_error(Riak::ProtobuffsFailedRequest) + + # disassociate + @bucket = @client.bucket(@index) + @bucket.props = {'yz_index' => nil} + + @client.delete_search_index(@index) + end +end \ No newline at end of file From 39acb988bbfdc0ebe7b5949701b28901ed11b61d Mon Sep 17 00:00:00 2001 From: Bryce Kerley Date: Tue, 27 Aug 2013 22:52:46 -0400 Subject: [PATCH 06/10] generate riak_search protobuffs too --- Rakefile | 11 +++++++- lib/riak/client/beefcake/messages.rb | 41 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 2e67245d..c33d6eb2 100644 --- a/Rakefile +++ b/Rakefile @@ -78,7 +78,12 @@ task :pb_defs => 'beefcake:pb_defs' namespace :beefcake do task :pb_defs => 'lib/riak/client/beefcake/messages.rb' - file 'lib/riak/client/beefcake/messages.rb' => %w{tmp/riak_kv.pb.rb tmp/riak_yokozuna.pb.rb} do |t| + task :clean do + sh "rm -rf tmp/riak_pb" + sh "rm -rf tmp/riak_kv.pb.rb tmp/riak_search.pb.rb tmp/riak_yokozuna.pb.rb" + end + + file 'lib/riak/client/beefcake/messages.rb' => %w{tmp/riak_kv.pb.rb tmp/riak_search.pb.rb tmp/riak_yokozuna.pb.rb} do |t| sh "cat lib/riak/client/beefcake/header tmp/riak.pb.rb #{t.prerequisites.join ' '} lib/riak/client/beefcake/footer > #{t.name}" end @@ -86,6 +91,10 @@ namespace :beefcake do sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/riak_kv.proto" end + file 'tmp/riak_search.pb.rb' => 'tmp/riak_pb' do |t| + sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/riak_search.proto" + end + file 'tmp/riak_yokozuna.pb.rb' => 'tmp/riak_pb' do |t| sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/riak_yokozuna.proto" end diff --git a/lib/riak/client/beefcake/messages.rb b/lib/riak/client/beefcake/messages.rb index 99e96e83..074caa0e 100644 --- a/lib/riak/client/beefcake/messages.rb +++ b/lib/riak/client/beefcake/messages.rb @@ -449,6 +449,47 @@ class RpbCounterGetReq class RpbCounterGetResp optional :value, :sint64, 1 end +## Generated from riak_search.proto for +require "beefcake" + + +class RpbSearchDoc + include Beefcake::Message +end + + +class RpbSearchQueryReq + include Beefcake::Message +end + + +class RpbSearchQueryResp + include Beefcake::Message +end + + +class RpbSearchDoc + repeated :fields, RpbPair, 1 +end + +class RpbSearchQueryReq + required :q, :bytes, 1 + required :index, :bytes, 2 + optional :rows, :uint32, 3 + optional :start, :uint32, 4 + optional :sort, :bytes, 5 + optional :filter, :bytes, 6 + optional :df, :bytes, 7 + optional :op, :bytes, 8 + repeated :fl, :bytes, 9 + optional :presort, :bytes, 10 +end + +class RpbSearchQueryResp + repeated :docs, RpbSearchDoc, 1 + optional :max_score, :float, 2 + optional :num_found, :uint32, 3 +end ## Generated from riak_yokozuna.proto for require "beefcake" From 9d0f694213f1275f2c53be8927cae29398f07fa3 Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Wed, 28 Aug 2013 14:22:02 -0700 Subject: [PATCH 07/10] Fix search message field name --- lib/riak/client/beefcake/messages.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/riak/client/beefcake/messages.rb b/lib/riak/client/beefcake/messages.rb index 074caa0e..8096ccfb 100644 --- a/lib/riak/client/beefcake/messages.rb +++ b/lib/riak/client/beefcake/messages.rb @@ -469,7 +469,7 @@ class RpbSearchQueryResp class RpbSearchDoc - repeated :fields, RpbPair, 1 + repeated :properties, RpbPair, 1 end class RpbSearchQueryReq From ce4cad1c00903b79b0a3933a8af8a3533aba4763 Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Wed, 28 Aug 2013 16:15:55 -0700 Subject: [PATCH 08/10] Overlay search message --- lib/riak/client/beefcake/message_overlay.rb | 10 ++++++++++ lib/riak/client/beefcake/messages.rb | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/riak/client/beefcake/message_overlay.rb b/lib/riak/client/beefcake/message_overlay.rb index 378983f0..3c2caac4 100644 --- a/lib/riak/client/beefcake/message_overlay.rb +++ b/lib/riak/client/beefcake/message_overlay.rb @@ -34,6 +34,16 @@ def has_postcommit=(newval) @postcommit ||= [] if newval end end + + class RpbSearchDoc + # rebuild the fields instance method since the + # generated :fields field overwrote this + def fields + self.class.fields + end + repeated :properties, RpbPair, 1 + end + end end end diff --git a/lib/riak/client/beefcake/messages.rb b/lib/riak/client/beefcake/messages.rb index 8096ccfb..074caa0e 100644 --- a/lib/riak/client/beefcake/messages.rb +++ b/lib/riak/client/beefcake/messages.rb @@ -469,7 +469,7 @@ class RpbSearchQueryResp class RpbSearchDoc - repeated :properties, RpbPair, 1 + repeated :fields, RpbPair, 1 end class RpbSearchQueryReq From 25b7816bec4de1fae8ef0c60e327f598f2617064 Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Wed, 18 Sep 2013 14:02:45 -0700 Subject: [PATCH 09/10] Move yokozuna specs into bryans integration template --- spec/integration/riak/yokozuna_spec.rb | 128 ---------------------- spec/integration/yokozuna/index_spec.rb | 70 +++++++++++- spec/integration/yokozuna/queries_spec.rb | 128 ++++++++++++++++++++-- spec/integration/yokozuna/schema_spec.rb | 59 +++++++++- 4 files changed, 236 insertions(+), 149 deletions(-) delete mode 100644 spec/integration/riak/yokozuna_spec.rb diff --git a/spec/integration/riak/yokozuna_spec.rb b/spec/integration/riak/yokozuna_spec.rb deleted file mode 100644 index 0ded77f5..00000000 --- a/spec/integration/riak/yokozuna_spec.rb +++ /dev/null @@ -1,128 +0,0 @@ -# require 'spec_helper' -$LOAD_PATH.unshift(File.dirname(__FILE__)) -$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', '..', '..', 'lib')) - -require 'rubygems' # Use the gems path only for the spec suite -require 'riak' -require 'rspec' - -RSpec.configure do |c| - # declare an exclusion filter - @http_port ||= 10018 # test_server.http_port - @client = Riak::Client.new(:http_port => @http_port) - yz_active = @client.ping rescue false - c.filter_run_excluding :yokozuna => !yz_active -end - -describe "Yokozuna", :yokozuna => true do - before(:all) do - @pbc_port ||= 10017 # test_server.pb_port - @http_port ||= 10018 # test_server.http_port - @client = Riak::Client.new(:http_port => @http_port, :pb_port => @pbc_port, :protocol => "pbc") - - @index = "test" - @schema = "testschema" - end - - context "using the admin client" do - it "should create / get / list indexes" do - @client.create_search_index(@index).should == true - # TODO: replace this with a wait function - sleep 1.1 - @client.get_search_index(@index).name.should == @index - @client.list_search_indexes.size.should >= 1 - lambda{ @client.get_search_index("herp_derp") }.should raise_error(Riak::ProtobuffsFailedRequest) - end - - it "should associate a bucket with an index" do - @bucket = Riak::Bucket.new(@client, @index) - @bucket.props = {'yz_index' => @index} - @bucket = @client.bucket(@index) - @bucket.props.should include('yz_index' => @index) - end - - it "should create / get schemas" do - content = <<-XML - - - - - - - - - - - - -_yz_id - - - - - XML - @client.create_search_schema(@schema, content) - schema_resp = @client.get_search_schema(@schema) - schema_resp.name.should == @schema - schema_resp.content.should == content - end - end - - context "using the search client" do - before(:all) do - @bucket = @client.bucket(@index) - - @object = @bucket.get_or_new("cat") - @object.raw_data = '{"cat_s":"Lela"}' - @object.content_type = 'application/json' - @object.store - - @object2 = @bucket.get_or_new("dogs") - @object2.raw_data = '{"dog_ss":["Einstein", "Olive"]}' - @object2.content_type = 'application/json' - @object2.store - - sleep 1.1 - end - - it "should search one row" do - resp = @client.search("test", "*:*", {:rows => 1}) - resp.should include('docs') - resp['docs'].size.should == 1 - end - - it "should search with df" do - resp = @client.search("test", "Olive", {:rows => 1, :df => 'dog_ss'}) - resp.should include('docs') - resp['docs'].size.should == 1 - resp['docs'].first['dog_ss'] - end - end - - # context "using the HTTP driver" do - # it "should search as XML" do - # resp = @http_client.search("test", "*:*", {:rows => 1, :wt => 'xml'}) - # root = REXML::Document.new(resp).root - # root.name.should == 'response' - # doc = root.elements['result'].elements['doc'] - # doc.should_not be_nil - # cats = doc.elements["str[@name='cat_s']"] - # cats.should have(1).item - # cats.text.should == 'Lela' - # end - # it "should search as JSON with df" do - # resp = @http_client.search("test", "Olive", {:rows => 1, :wt => 'json', :df => 'dog_ss'}) - # end - # end - - after(:all) do - # Can't delete index with associate buckets - lambda{ @client.delete_search_index(@index) }.should raise_error(Riak::ProtobuffsFailedRequest) - - # disassociate - @bucket = @client.bucket(@index) - @bucket.props = {'yz_index' => nil} - - @client.delete_search_index(@index) - end -end \ No newline at end of file diff --git a/spec/integration/yokozuna/index_spec.rb b/spec/integration/yokozuna/index_spec.rb index 344067ed..7704fb43 100644 --- a/spec/integration/yokozuna/index_spec.rb +++ b/spec/integration/yokozuna/index_spec.rb @@ -1,16 +1,74 @@ -describe "Yokozuna" do +require 'spec_helper' +require 'riak' + +describe "Yokozuna", test_server: true, integration: true do + before(:all) do + opts = { + http_port: test_server.http_port, + pb_port: test_server.pb_port, + protocol: 'pbc' + } + test_server.start + @client = Riak::Client.new opts + + @index = "test" + @schema = "testschema" + end + context "without any indexes" do - it "should allow index creation" + it "should allow index creation" do + @client.create_search_index(@index).should == true + end end context "with an index" do - it "should have an index list" - it "should allow index inspection" + before :all do + @client.create_search_index(@index).should == true + wait_until{ !@client.get_search_index(@index).nil? } + end - it "should associate an index with a bucket" + it "should allow index inspection" do + @client.get_search_index(@index).name.should == @index + lambda{ @client.get_search_index("herp_derp") }.should raise_error(Riak::ProtobuffsFailedRequest) + end + + it "should have an index list" do + @client.list_search_indexes.size.should >= 1 + end + + it "should associate a bucket with an index" do + @bucket = Riak::Bucket.new(@client, @index) + @bucket.props = {'yz_index' => @index} + @bucket = @client.bucket(@index) + @bucket.props.should include('yz_index' => @index) + end context "associated with a bucket" do - it "should index on object writes" + before :all do + @bucket = Riak::Bucket.new(@client, @index) + @bucket.props = {'yz_index' => @index} + @bucket = @client.bucket(@index) + @bucket.props.should include('yz_index' => @index) + end + + it "should index on object writes" do + object = @bucket.get_or_new("cat") + object.raw_data = {"cat_s"=>"Lela"}.to_json + object.content_type = 'application/json' + object.store + sleep 1.1 # pause for index commit to trigger + + resp = @client.search(@index, "cat_s:Lela") + resp.should include('docs') + resp['docs'].size.should == 1 + end end end + + def wait_until(attempts=5) + begin + break if yield rescue nil + sleep 1 + end while (attempts -= 1) > 0 + end end diff --git a/spec/integration/yokozuna/queries_spec.rb b/spec/integration/yokozuna/queries_spec.rb index afe4c8da..b7a142c5 100644 --- a/spec/integration/yokozuna/queries_spec.rb +++ b/spec/integration/yokozuna/queries_spec.rb @@ -1,14 +1,118 @@ -describe "Yokozona queries with a schema and indexes" do - it "should produce results on single term queries" - it "should produce results on multiple term queries" - it "should produce results on queries with boolean logic" - it "should produce results on range queries" - it "should produce results on phrase queries" - it "should produce results on wildcard queries" - it "should produce results on regexp queries" - - context "using parameters" do - it "should support start/offset parameters" - it "should support rows/limit parameters" +require 'spec_helper' +require 'riak' + +describe "Yokozona queries", test_server: true, integration: true do + before :all do + opts = { + http_port: test_server.http_port, + pb_port: test_server.pb_port, + protocol: 'pbc' + } + test_server.start + @client = Riak::Client.new opts + end + + context "with a schema and indexes" do + before :all do + @index = "test" + + @client.create_search_index(@index).should == true + sleep 1.1 # wait for index to load + @bucket = Riak::Bucket.new(@client, @index) + @bucket.props = {'yz_index' => @index} + + # populate objects + def build_json_obj(key, data) + object = @bucket.get_or_new(key) + object.raw_data = data.to_json + object.content_type = 'application/json' + object.store + object + end + + @o1 = build_json_obj("cat", {"cat_s"=>"Lela"}) + @o2 = build_json_obj("docs", {"dog_ss"=>["Einstein", "Olive"]}) + build_json_obj("Z", {"username_s"=>"Z", "name_s"=>"ryan", "age_i"=>30}) + build_json_obj("R", {"username_s"=>"R", "name_s"=>"eric", "age_i"=>34}) + build_json_obj("F", {"username_s"=>"F", "name_s"=>"bryan fink", "age_i"=>32}) + build_json_obj("H", {"username_s"=>"H", "name_s"=>"brett", "age_i"=>14}) + + sleep 1.1 # pause for index commit to trigger + end + + it "should produce results on single term queries" do + resp = @client.search("test", "username_s:Z") + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should produce results on multiple term queries" do + resp = @client.search("test", "username_s:(F OR H)") + resp.should include('docs') + resp['docs'].size.should == 2 + end + + it "should produce results on queries with boolean logic" do + resp = @client.search("test", "username_s:Z AND name_s:ryan") + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should produce results on range queries" do + resp = @client.search("test", "age_i:[30 TO 33]") + resp.should include('docs') + resp['docs'].size.should == 2 + end + + it "should produce results on phrase queries" do + resp = @client.search("test", 'name_s:"bryan fink"') + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should produce results on wildcard queries" do + resp = @client.search("test", "name_s:*ryan*") + resp.should include('docs') + resp['docs'].size.should == 2 + end + + it "should produce results on regexp queries" do + resp = @client.search("test", "name_s:/br.*/") + resp.should include('docs') + resp['docs'].size.should == 2 + end + + context "using parameters" do + it "should search one row" do + resp = @client.search("test", "*:*", {:rows => 1}) + resp.should include('docs') + resp['docs'].size.should == 1 + end + + it "should search with df" do + resp = @client.search("test", "Olive", {:rows => 1, :df => 'dog_ss'}) + resp.should include('docs') + resp['docs'].size.should == 1 + resp['docs'].first['dog_ss'] + end + + it "should produce top result on sort" do + resp = @client.search("test", "username_s:*", {:sort => "age_i asc"}) + resp.should include('docs') + resp['docs'].first['age_i'].to_i.should == 14 + end + + end + + after(:all) do + # Can't delete index with associate buckets + lambda{ @client.delete_search_index(@index) }.should raise_error(Riak::ProtobuffsFailedRequest) + + # disassociate + @bucket = @client.bucket(@index) + @bucket.props = {'yz_index' => nil} + + @client.delete_search_index(@index) + end end end diff --git a/spec/integration/yokozuna/schema_spec.rb b/spec/integration/yokozuna/schema_spec.rb index 47ffe1da..56b6230b 100644 --- a/spec/integration/yokozuna/schema_spec.rb +++ b/spec/integration/yokozuna/schema_spec.rb @@ -1,8 +1,61 @@ -describe 'Yokozuna' do +require 'spec_helper' +require 'riak' + +describe "Yokozuna", test_server: true, integration: true do + before(:all) do + opts = { + http_port: test_server.http_port, + pb_port: test_server.pb_port, + protocol: 'pbc' + } + test_server.start + @client = Riak::Client.new opts + + @index = "test" + @schema = "testschema" + end + context 'with no schema' do - it 'should allow schema creation' + it 'should allow schema creation' do + @client.create_search_schema(@schema, SCHEMA_CONTENT) + wait_until{ !@client.get_search_schema(@schema).nil? } + @client.get_search_schema(@schema).should_not be_nil + end end context 'with a schema' do - it 'should have a readable schema' + it 'should have a readable schema' do + @client.create_search_schema(@schema, SCHEMA_CONTENT) + wait_until{ !@client.get_search_schema(@schema).nil? } + schema_resp = @client.get_search_schema(@schema) + schema_resp.name.should == @schema + schema_resp.content.should == SCHEMA_CONTENT + end + end + + SCHEMA_CONTENT = <<-XML + + + + + + + + + + + + +_yz_id + + + + + XML + + def wait_until(attempts=5) + begin + break if yield rescue nil + sleep 1 + end while (attempts -= 1) > 0 end end From d12842124292153d35c6f7dd5d8812f271a16d29 Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Thu, 19 Sep 2013 14:04:38 -0700 Subject: [PATCH 10/10] kickoff incomplete utf8 test --- spec/integration/yokozuna/queries_spec.rb | 69 ++++++++++++++--------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/spec/integration/yokozuna/queries_spec.rb b/spec/integration/yokozuna/queries_spec.rb index b7a142c5..4c128846 100644 --- a/spec/integration/yokozuna/queries_spec.rb +++ b/spec/integration/yokozuna/queries_spec.rb @@ -1,3 +1,4 @@ +# encoding: UTF-8 require 'spec_helper' require 'riak' @@ -17,87 +18,87 @@ @index = "test" @client.create_search_index(@index).should == true - sleep 1.1 # wait for index to load + wait_until{ !@client.get_search_index(@index).nil? } @bucket = Riak::Bucket.new(@client, @index) @bucket.props = {'yz_index' => @index} - # populate objects - def build_json_obj(key, data) - object = @bucket.get_or_new(key) - object.raw_data = data.to_json - object.content_type = 'application/json' - object.store - object - end - - @o1 = build_json_obj("cat", {"cat_s"=>"Lela"}) - @o2 = build_json_obj("docs", {"dog_ss"=>["Einstein", "Olive"]}) - build_json_obj("Z", {"username_s"=>"Z", "name_s"=>"ryan", "age_i"=>30}) - build_json_obj("R", {"username_s"=>"R", "name_s"=>"eric", "age_i"=>34}) - build_json_obj("F", {"username_s"=>"F", "name_s"=>"bryan fink", "age_i"=>32}) - build_json_obj("H", {"username_s"=>"H", "name_s"=>"brett", "age_i"=>14}) + @o1 = build_json_obj(@bucket, "cat", {"cat_s"=>"Lela"}) + @o2 = build_json_obj(@bucket, "docs", {"dog_ss"=>["Einstein", "Olive"]}) + build_json_obj(@bucket, "Z", {"username_s"=>"Z", "name_s"=>"ryan", "age_i"=>30}) + build_json_obj(@bucket, "R", {"username_s"=>"R", "name_s"=>"eric", "age_i"=>34}) + build_json_obj(@bucket, "F", {"username_s"=>"F", "name_s"=>"bryan fink", "age_i"=>32}) + build_json_obj(@bucket, "H", {"username_s"=>"H", "name_s"=>"brett", "age_i"=>14}) sleep 1.1 # pause for index commit to trigger end it "should produce results on single term queries" do - resp = @client.search("test", "username_s:Z") + resp = @client.search(@index, "username_s:Z") resp.should include('docs') resp['docs'].size.should == 1 end it "should produce results on multiple term queries" do - resp = @client.search("test", "username_s:(F OR H)") + resp = @client.search(@index, "username_s:(F OR H)") resp.should include('docs') resp['docs'].size.should == 2 end it "should produce results on queries with boolean logic" do - resp = @client.search("test", "username_s:Z AND name_s:ryan") + resp = @client.search(@index, "username_s:Z AND name_s:ryan") resp.should include('docs') resp['docs'].size.should == 1 end it "should produce results on range queries" do - resp = @client.search("test", "age_i:[30 TO 33]") + resp = @client.search(@index, "age_i:[30 TO 33]") resp.should include('docs') resp['docs'].size.should == 2 end it "should produce results on phrase queries" do - resp = @client.search("test", 'name_s:"bryan fink"') + resp = @client.search(@index, 'name_s:"bryan fink"') resp.should include('docs') resp['docs'].size.should == 1 end it "should produce results on wildcard queries" do - resp = @client.search("test", "name_s:*ryan*") + resp = @client.search(@index, "name_s:*ryan*") resp.should include('docs') resp['docs'].size.should == 2 end it "should produce results on regexp queries" do - resp = @client.search("test", "name_s:/br.*/") + resp = @client.search(@index, "name_s:/br.*/") resp.should include('docs') resp['docs'].size.should == 2 end + # TODO: run this when pb utf8 works + it "should support utf8" do + build_json_obj(@bucket, "ja", {"text_ja"=>"私はハイビスカスを食べるのが 大好き"}) + # sleep 1.1 # pause for index commit to trigger + # resp = @client.search(@index, "text_ja:大好き") + # resp.should include('docs') + # resp['docs'].size.should == 1 + end + context "using parameters" do it "should search one row" do - resp = @client.search("test", "*:*", {:rows => 1}) + resp = @client.search(@index, "*:*", {:rows => 1}) resp.should include('docs') resp['docs'].size.should == 1 end it "should search with df" do - resp = @client.search("test", "Olive", {:rows => 1, :df => 'dog_ss'}) + resp = @client.search(@index, "Olive", {:rows => 1, :df => 'dog_ss'}) resp.should include('docs') resp['docs'].size.should == 1 resp['docs'].first['dog_ss'] end it "should produce top result on sort" do - resp = @client.search("test", "username_s:*", {:sort => "age_i asc"}) + resp = @client.search(@index, "username_s:*", {:sort => "age_i asc"}) resp.should include('docs') resp['docs'].first['age_i'].to_i.should == 14 end @@ -115,4 +116,20 @@ def build_json_obj(key, data) @client.delete_search_index(@index) end end + + def wait_until(attempts=5) + begin + break if yield rescue nil + sleep 1 + end while (attempts -= 1) > 0 + end + + # populate objects + def build_json_obj(bucket, key, data) + object = bucket.get_or_new(key) + object.raw_data = data.to_json + object.content_type = 'application/json' + object.store + object + end end