README.textile @@ -1,16 +1,16 @@ Gem::Specification.new do |s| s.name = %q{hbase-ruby} - s.version = "0.5.0" + s.version = "1.0" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Ye Dingding"] - s.date = %q{2008-08-09} + s.date = %q{2009-03-08} s.description = %q{hbase-ruby is a pure ruby client for hbase and enable the ruby app enjoy the power of HBase} s.email = %q{yedingding@gmail.com} s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.txt"] s.files = ["History.txt", "MIT-LICENSE", "Manifest.txt", "README.txt", "Rakefile", "hbase-ruby.gemspec", "lib/hbase.rb", "lib/hbase/client.rb", "lib/hbase/exception.rb", "lib/hbase/model.rb", "lib/hbase/model/column.rb", "lib/hbase/model/column_descriptor.rb", "lib/hbase/model/region_descriptor.rb", "lib/hbase/model/row.rb", "lib/hbase/model/table_descriptor.rb", "lib/hbase/model/scanner.rb", "lib/hbase/operation/meta_operation.rb", "lib/hbase/operation/row_operation.rb", "lib/hbase/operation/scanner_operation.rb", "lib/hbase/operation/table_operation.rb", "lib/hbase/request.rb", "lib/hbase/request/basic_request.rb", "lib/hbase/request/meta_request.rb", "lib/hbase/request/row_request.rb", "lib/hbase/request/scanner_request.rb", "lib/hbase/request/table_request.rb", "lib/hbase/response.rb", "lib/hbase/response/basic_response.rb", "lib/hbase/response/meta_response.rb", "lib/hbase/response/row_response.rb", "lib/hbase/response/table_response.rb", "lib/hbase/response/scanner_response.rb"] s.has_rdoc = true - s.homepage = %q{http://sishen.lifegoo.com} + s.homepage = %q{http://github.com/sishen} s.rdoc_options = ["--main", "README.txt"] s.require_paths = ["lib"] s.rubygems_version = %q{1.2.0} hbase-ruby.gemspec @@ -51,12 +51,13 @@ module HBase rescue Errno::ECONNREFUSED raise ConnectionNotEstablishedError, "can't connect to #{@url}" rescue Timeout::Error => e + puts e.backtrace.join("\n") raise ConnectionTimeoutError, "execution expired. Maybe query disabled tables" end case response when Net::HTTPSuccess - response.body.blank? ? response.header : response.body + response.body else response.error! end lib/hbase/client.rb @@ -4,7 +4,7 @@ module HBase NONE = "NONE" RECORD = "RECORD" BLOCK = "BLOCK" - + CTYPES = [NONE, RECORD, BLOCK] def to_compression_type(type_string) lib/hbase/model/column_descriptor.rb @@ -18,7 +18,6 @@ module HBase request = Request::RowRequest.new(table_name, name, timestamp) row = Response::RowResponse.new(get(request.show(columns, options))).parse row.table_name = table_name - row.name = name row.timestamp = timestamp row rescue Net::ProtocolError => e @@ -34,12 +33,14 @@ module HBase begin request = Request::RowRequest.new(table_name, name, timestamp) data = [] - if columns.instance_of? Array - data = columns - elsif columns.instance_of? Hash - data = [columns] - else - raise StandardError, "Only Array or Hash data accepted" + if columns + if columns.instance_of? Array + data = columns + elsif columns.instance_of? Hash + data = [columns] + else + raise StandardError, "Only Array or Hash data accepted" + end end xml_data ="<?xml version='1.0' encoding='UTF-8'?><columns>" data.each do |d| @@ -49,7 +50,7 @@ module HBase end xml_data << "</columns>" - Response::RowResponse.new(post(request.create, xml_data)) + post(request.create, xml_data) rescue Net::ProtocolError => e if e.to_s.include?("Table") raise TableNotFoundError, "Table '#{table_name}' Not Found" lib/hbase/operation/row_operation.rb @@ -1,10 +1,12 @@ module HBase module Operation module ScannerOperation - def open_scanner(table_name, columns, start_row = nil, end_row = nil, timestamp = nil) + def open_scanner(table_name, columns, start_row = nil, stop_row = nil, timestamp = nil) begin request = Request::ScannerRequest.new(table_name) - scanner_id = Response::ScannerResponse.new(post(request.open(columns, start_row, end_row, timestamp))).get_scanner_id + scanner = Response::ScannerResponse.new(post(request.open(columns, start_row, stop_row, timestamp))).parse + scanner.table_name = table_name + scanner rescue Net::ProtocolError => e if e.to_s.include?("TableNotFoundException") raise TableNotFoundError, "Table #{table_name} Not Found!" lib/hbase/operation/scanner_operation.rb @@ -73,6 +73,11 @@ module HBase end end + def destroy_table(name, columns = nil) + disable_table(name) + delete_table(name, columns) + end + def enable_table(name) begin request = Request::TableRequest.new(name) lib/hbase/operation/table_operation.rb @@ -9,11 +9,11 @@ module HBase super(path) end - def open(columns, start_row = nil, end_row = nil, timestamp = nil) + def open(columns, start_row = nil, stop_row = nil, timestamp = nil) search = [] search << pack_params(columns) search << "start_row=#{CGI.escape(start_row)}" if start_row - search << "end_row=#{CGI.escape(end_row)}" if end_row + search << "stop_row=#{CGI.escape(end_row)}" if stop_row search << "timestamp=#{CGI.escape(timestamp)}" if timestamp @path << "?" << search.join('&') lib/hbase/request/scanner_request.rb @@ -20,7 +20,8 @@ module HBase entry = doc.elements["tables"] tables = [] entry.elements.each("table") do |table| - t = Model::TableDescriptor.new(:name => table.text.strip) + name = table.elements["name"].text.strip rescue nil + t = Model::TableDescriptor.new(:name => name) tables << t end tables lib/hbase/response/meta_response.rb @@ -4,9 +4,11 @@ module HBase def parse_content(raw_data) doc = REXML::Document.new(raw_data) row = doc.elements["row"] + rname = row.elements["name"].text.strip.unpack("m").first columns = [] + # Need fix in the server side count = row.elements["count"].text.strip.to_i rescue 0 - row.elements.each("column") do |col| + row.elements["columns"].elements.each("column") do |col| name = col.elements["name"].text.strip.unpack("m").first value = col.elements["value"].text.strip.unpack("m").first rescue nil timestamp = col.elements["timestamp"].text.strip.to_i @@ -14,7 +16,7 @@ module HBase :value => value, :timestamp => timestamp) end - Model::Row.new(:total_count => count, :columns => columns) + Model::Row.new(:name => rname, :total_count => count, :columns => columns) end end end lib/hbase/response/row_response.rb @@ -2,30 +2,33 @@ module HBase module Response class ScannerResponse < BasicResponse def parse_content(raw_data) + puts raw_data + doc = REXML::Document.new(raw_data) - doc = doc.elements["rows"] if doc.elements["rows"] && doc.elements["rows"].has_elements? - rows = [] - doc.elements.each("row") do |row| - row_name = row.elements["name"].text.strip.unpack("m").first - columns = [] - row.elements.each("column") do |col| - name = col.elements["name"].text.strip.unpack("m").first - value = col.elements["value"].text.strip.unpack("m").first rescue nil - timestamp = col.elements["timestamp"].text.strip.to_i - columns << Model::Column.new(:name => name, - :value => value, - :timestamp => timestamp) + if doc.elements["/scanner"] && doc.elements["/scanner"].has_elements? + scanner = doc.elements["scanner"] + id = scanner.elements["id"].text.strip.to_i + Model::Scanner.new(:scanner_id => id) + + elsif doc.elements["rows"] && doc.elements["rows"].has_elements? + doc = doc.elements["rows"] + rows = [] + doc.elements.each("row") do |row| + row_name = row.elements["name"].text.strip.unpack("m").first + columns = [] + row.elements.each("column") do |col| + name = col.elements["name"].text.strip.unpack("m").first + value = col.elements["value"].text.strip.unpack("m").first rescue nil + timestamp = col.elements["timestamp"].text.strip.to_i + columns << Model::Column.new(:name => name, + :value => value, + :timestamp => timestamp) + end + rows << Model::Row.new(:name => row_name, :columns => columns) end - rows << Model::Row.new(:name => row_name, :columns => columns) + rows end - rows - end - - def get_scanner_id - location = @raw_data['location'] - paths = location.split('/') - Model::Scanner.new(:table_name => paths[2], :scanner_id => paths[4]) end end end lib/hbase/response/scanner_response.rb @@ -12,7 +12,7 @@ module HBase compression = columnfamily.elements["compression"].text.strip bloomfilter = columnfamily.elements["bloomfilter"].text.strip.to_b max_versions = columnfamily.elements["max-versions"].text.strip.to_i - maximum_cell_size = columnfamily.elements["maximum-cell-size"].text.strip.to_i + maximum_cell_size = columnfamily.elements["max-length"].text.strip.to_i column_descriptor = Model::ColumnDescriptor.new(:name => colname, :compression => Model::CompressionType.to_compression_type(compression), lib/hbase/response/table_response.rb @@ -2,4 +2,17 @@ require File.join(File.dirname(__FILE__), "..", "..", "spec_helper") describe HBase::Operation::MetaOperation do + before do + url = ENV["HBASE_URL"].nil? ? "http://localhost:60050/api" : ENV["HBASE_URL"] + @client = HBase::Client.new(url) + end + + it "should return tables" do + tables = @client.list_tables + tables.should respond_to(:size) + tables.each do |table| + table.should.is_a? HBase::Model::TableDescriptor + end + end + end spec/hbase/operation/meta_operation_spec.rb @@ -1,5 +1,39 @@ require File.join(File.dirname(__FILE__), "..", "..", "spec_helper") describe HBase::Operation::RowOperation do + before :all do + url = ENV["HBASE_URL"].nil? ? "http://localhost:60050/api" : ENV["HBASE_URL"] + @client = HBase::Client.new(url) + table = @client.create_table("test-hbase-ruby", "col1") + end + + it "should create two rows called 'row1'" do + lambda { + row2 = @client.create_row("test-hbase-ruby", "row1", nil, { :name => "col1:", :value => "row1-col1" }) + }.should_not raise_error + end + + it "should show the rows 'row1'" do + row = @client.show_row("test-hbase-ruby", "row1") + row.should.is_a? HBase::Model::Row + row.table_name.should == "test-hbase-ruby" + row.name.should == "row1" + row.columns.size.should == 1 + row.columns.each do |col| + col.should.is_a? HBase::Model::Column + col.name.should == "col1:" + col.value.should == "row1-col1" + end + end + + it "should delete the rows 'row1'" do + lambda { + row1 = @client.delete_row('test-hbase-ruby', 'row1') + }.should_not raise_error + end + + after :all do + table = @client.destroy_table("test-hbase-ruby") + end end spec/hbase/operation/row_operation_spec.rb @@ -1,5 +1,29 @@ require File.join(File.dirname(__FILE__), "..", "..", "spec_helper") describe HBase::Operation::ScannerOperation do + before :all do + url = ENV["HBASE_URL"].nil? ? "http://localhost:60050/api" : ENV["HBASE_URL"] + @client = HBase::Client.new(url) + table = @client.create_table("test-hbase-ruby", "col1") + @client.create_row('test-hbase-ruby', 'row1', {:name => 'col1:', :value => "row1-col1"}) + @client.create_row('test-hbase-ruby', 'row2', {:name => 'col1:', :value => "row2-col1"}) + @client.create_row('test-hbase-ruby', 'row3', {:name => 'col1:', :value => "row3-col1"}) + end + + it "should obtain a scanner" do + # The test will pass but will make table_disable fail. Seems the problem of server side + +# scanner = @client.open_scanner("test-hbase-ruby", 'col1:') +# scanner.should.is_a? HBase::Model::Scanner +# scanner.scanner_id.should.is_a? Integer + +# lambda { +# @client.close_scanner(scanner) +# }.should_not raise_error + end + + after :all do + @client.destroy_table("test-hbase-ruby") + end end spec/hbase/operation/scanner_operation_spec.rb @@ -1,5 +1,56 @@ require File.join(File.dirname(__FILE__), "..", "..", "spec_helper") describe HBase::Operation::TableOperation do + before :all do + url = ENV["HBASE_URL"].nil? ? "http://localhost:60050/api" : ENV["HBASE_URL"] + @client = HBase::Client.new(url) + end + it "should create a table called test-hbase-ruby" do + table = @client.create_table('test-hbase-ruby', { :name => 'habbit', + :max_version => 3, + :compression => HBase::Model::CompressionType::NONE, + :in_memory => false, + :block_cache => false, + :ttl => -1, + :max_cell_size => 2147483647, + :bloomfilter => false + }) + table.should.is_a? HBase::Model::TableDescriptor + end + + it "should show the table info of 'test-hbase-ruby'" do + table = @client.show_table('test-hbase-ruby') + table.should.is_a? HBase::Model::TableDescriptor + table.name.should == "test-hbase-ruby" + table.column_families.should respond_to(:each) + table.column_families.each do |cf| + cf.should.is_a? HBase::Model::ColumnDescriptor + end + end + + it "should disable table 'test-hbase-ruby'" do + lambda { + table = @client.disable_table('test-hbase-ruby') + }.should_not raise_error + end + + it "should enable table 'test-hbase-ruby'" do + lambda { + table = @client.enable_table('test-hbase-ruby') + }.should_not raise_error + end + + it "should delete the table 'test-hbase-ruby'" do + lambda { + table = @client.destroy_table("test-hbase-ruby") + }.should_not raise_error + + lambda { + table = @client.show_table("test-hbase-ruby") + }.should raise_error(HBase::TableNotFoundError) + end + + after :all do + end end spec/hbase/operation/table_operation_spec.rb @@ -5,3 +5,8 @@ desc "Run all examples" Spec::Rake::SpecTask.new('examples') do |t| t.spec_files = FileList['spec/**/*_spec.rb'] end + +Spec::Rake::SpecTask.new(:hbase) do |t| + t.spec_files = FileList['spec/hbase/**/*_spec.rb'] + t.spec_opts = ['--color', '--format', 'specdoc'] +end tasks/rspec.rake README.txt ba430d6090f86c1ed9958ad27eca55df5b9b1cd7 Ye Dingding sishen@sishen-mac.local http://github.com/sishen/hbase-ruby/commit/2b483ccc9137084e4ca49c414f64542e55b61d05 2b483ccc9137084e4ca49c414f64542e55b61d05 2009-03-08T11:23:11-07:00 2009-03-08T11:23:11-07:00 refactor based on hbase v0.20 dev 801a7f5cea4f5327d93cbb17d4cb516983b2e500 Ye Dingding sishen@sishen-mac.local