From 61974f46d3485686ab496f1a4f5479081c1e5d13 Mon Sep 17 00:00:00 2001 From: nas Date: Sun, 20 Jun 2010 09:40:01 +0100 Subject: [PATCH] initial commit with comprehensive select operations --- History.txt | 4 + Manifest.txt | 7 ++ README.rdoc | 104 ++++++++++++++++++ lib/yql.rb | 11 ++ lib/yql/client.rb | 65 ++++++++++++ lib/yql/error.rb | 19 ++++ lib/yql/query_builder.rb | 186 +++++++++++++++++++++++++++++++++ spec/spec_helper.rb | 3 + spec/yql/client_spec.rb | 9 ++ spec/yql/query_builder_spec.rb | 9 ++ yql.gemspec | 29 +++++ 11 files changed, 446 insertions(+) create mode 100644 History.txt create mode 100644 Manifest.txt create mode 100644 README.rdoc create mode 100644 lib/yql.rb create mode 100644 lib/yql/client.rb create mode 100644 lib/yql/error.rb create mode 100644 lib/yql/query_builder.rb create mode 100644 spec/spec_helper.rb create mode 100644 spec/yql/client_spec.rb create mode 100644 spec/yql/query_builder_spec.rb create mode 100644 yql.gemspec diff --git a/History.txt b/History.txt new file mode 100644 index 0000000..20372d1 --- /dev/null +++ b/History.txt @@ -0,0 +1,4 @@ +=== 0.1.1 2010-06-20 + +* 1 major enhancement: + * Initial release \ No newline at end of file diff --git a/Manifest.txt b/Manifest.txt new file mode 100644 index 0000000..39a9c46 --- /dev/null +++ b/Manifest.txt @@ -0,0 +1,7 @@ +History.txt +Manifest.txt +README.rdoc +lib/yql.rb +lib/yql/client.rb +lib/yql/error.rb +lib/yql/query_builder.rb \ No newline at end of file diff --git a/README.rdoc b/README.rdoc new file mode 100644 index 0000000..c9ebd65 --- /dev/null +++ b/README.rdoc @@ -0,0 +1,104 @@ +==DESCRIPTION + +A basic Ruby Wrapper for interacting programatically with YQL API. + +I started working on this library during ScienceHackDay held at Guardian, London on 19/06/2010-20/06/2010 + +==TODO + +1. Add Unit Tests +2. Add oauth +3. Add table creation +4. Add update / insert / delete operations + +==INSTALLATION + +sudo gem source --add http://rubygems.org + +sudo gem install yql + + +==USAGE + +require 'rubygems' + +require 'yql' + +===Building Query + + +====Finders + +yql = Yql::Client.new + +query = Yql::QueryBuilder.new 'yelp.review.search' + +query.to_s #=> "select * from yelp.review.search" + +query.find #=> "select * from yelp.review.search limit 1" + +query.limit = 4 + +query.to_s #=> "select * from yelp.review.search limit 4" + +query.find_all #=> "select * from yelp.review.search" + + +====Conditions + +query.conditions = "term like '%pizza%'" + +query.to_s #=> "select * from yelp.review.search where term='%pizza%'" + +query.conditions = {:term => 'pizza'} + +query.to_s #=> "select * from yelp.review.search where term = 'pizza'" + +query.conditions = {:term => 'pizza', :location => 'london', 'ywsid' => '6L0Lc-yn1OKMkCKeXLD4lg'} + +query.to_s #=> "select * from yelp.review.search where term='pizza' and location='london' and ywsid= '6L0Lc-yn1OKMkCKeXLD4lg'" + +query.select = 'user_photo_url, state' + +yql.query = query +result = yql.get + +yql.format = 'json' +result = yql.get + + +===Piped Filters + +query.unique = 'name' + +query.to_s #=> "select title, Rating, LastReviewIntro from yelp.review.search where ywsid='6L0Lc-yn1OKMkCKeXLD4lg' and term='pizza' and location='london' | unique(field='name')" + +query.tail = 4 + +query.to_s #=> "select title, Rating, LastReviewIntro from yelp.review.search where ywsid='6L0Lc-yn1OKMkCKeXLD4lg' and term='pizza' and location='london' | unique(field='name') | tail(count=4)" + +query.reorder_pipe_command :from => 1, :to => 0 + +query.to_s #=> "select title, Rating, LastReviewIntro from yelp.review.search where ywsid='6L0Lc-yn1OKMkCKeXLD4lg' and term='pizza' and location='london' | tail(count=4) | unique(field='name')" + +yql.format = 'json' +result = yql.get + + +====Pagination + +query.per_page = 10 +query.current_page = 1 + +yql.query = query +result = yql.get + + +===Describe and show tables + +query = Yql::QueryBuilder.describe_table('yelp.review.search') + +query = Yql::QueryBuilder.show_tables + +yql.query = query +result = yql.get diff --git a/lib/yql.rb b/lib/yql.rb new file mode 100644 index 0000000..c68a9b8 --- /dev/null +++ b/lib/yql.rb @@ -0,0 +1,11 @@ +$:.unshift(File.dirname(__FILE__)) unless + $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__))) + +require 'rubygems' +require 'CGI' +require 'net/http' +require 'net/https' +require 'rexml/document' +require 'yql/error.rb' +require 'yql/client.rb' +require 'yql/query_builder.rb' \ No newline at end of file diff --git a/lib/yql/client.rb b/lib/yql/client.rb new file mode 100644 index 0000000..571d956 --- /dev/null +++ b/lib/yql/client.rb @@ -0,0 +1,65 @@ +require 'net/http' +module Yql + + class Client + + BASE_URL = 'query.yahooapis.com' + VERSION = 'v1' + URL_SUFFIX = 'public/yql' + YQL_ENV = 'http://datatables.org/alltables.env' + + attr_accessor :query, :diagnostics, :format + + def initialize(args={}) + @diagnostics = args[:diagnostics] #true or false + @version = args[:version] + @query = args[:query] + @format = args[:format] || 'xml' + end + + def query + @query.kind_of?(Yql::QueryBuilder) ? @query.to_s : @query + end + + def version + @version ||= VERSION + end + + def full_url + "#{version}/#{URL_SUFFIX}" + end + + def get + if query.nil? + raise Yql::IncompleteRequestParameter, "Query not specified" + end + http = Net::HTTP.new(BASE_URL, Net::HTTP.https_default_port) + http.use_ssl = true + path = "/#{version}/#{URL_SUFFIX}" + result = http.post(path, parameters) + #raise(Yql::ResponseFailure, result.response) unless result.code == '200' + return result.body unless format == 'xml' + REXML::Document.new(result.body) + end + + def parameters + url_parameters = "q=#{CGI.escape(query)}&env=#{YQL_ENV}" + url_parameters = add_format(url_parameters) + add_diagnostics(url_parameters) + end + + def add_format(existing_parameters) + return unless existing_parameters + return existing_parameters unless format + return existing_parameters + "&format=#{format}" + end + + def add_diagnostics(existing_parameters) + return unless existing_parameters + return existing_parameters unless diagnostics + return existing_parameters + "&diagnostics=true" + end + + end + +end diff --git a/lib/yql/error.rb b/lib/yql/error.rb new file mode 100644 index 0000000..735dbb7 --- /dev/null +++ b/lib/yql/error.rb @@ -0,0 +1,19 @@ +module Yql + class Error < StandardError + + def initialize(data) + @data = data + super + end + end + + class ResponseFailure < Error + end + + class NoResult < Error + end + + class IncompleteRequestParameter < Error + end + +end \ No newline at end of file diff --git a/lib/yql/query_builder.rb b/lib/yql/query_builder.rb new file mode 100644 index 0000000..f4c6b2e --- /dev/null +++ b/lib/yql/query_builder.rb @@ -0,0 +1,186 @@ +module Yql + + class QueryBuilder + + attr_accessor :table, :conditions, :limit, :truncate, :sanitize_field, :select, + :sort_field, :current_pipe_command_types, :sort_descending, + :per_page, :current_page + + def initialize(table, args = {}) + @select = args[:select] + @table = table + @use_statement = args[:use] + @conditions = args[:conditions] + @limit = args[:limit] + @tail = args[:tail] + @reverse = args[:reverse] + @unique = args[:unique] + @sanitize = args[:sanitize] + @sort_descending = args[:sort_descending] + @sanitize_field = args[:sanitize_field] + @sort_field = args[:sort_field] + @current_pipe_command_types = [] + @per_page = args[:per_page] + end + + def find + self.limit = 1 + "#{construct_query}" + end + + def find_all + self.limit = nil + construct_query + end + + def self.show_tables + "show tables;" + end + + def self.describe_table(table) + "desc #{table};" + end + + def describe_table + Yql::QueryBuilder.describle_table(table) + end + + def to_s + construct_query + end + + def limit + return unless @limit + "limit #{@limit}" + end + + # Conditions can either be provided as hash or plane string + def conditions + if @conditions.kind_of?(String) + cond = @conditions + elsif @conditions.kind_of?(Hash) + cond = @conditions.collect do |k,v| + val = v.kind_of?(String) ? "'#{v}'" : v + "#{k.to_s}=#{val}" + end + cond = cond.join(' and ') + else + return + end + return "where #{cond}" + end + + %w{sort tail truncate reverse unique sanitize}.each do |method| + self.send(:define_method, "#{method}=") do |param| + instance_variable_set("@#{method}", param) + current_pipe_command_types << method unless current_pipe_command_types.include?(method) + end + end + + # Cption can be piped + # Sorts the result set according to the specified field (column) in the result set. + def sort + return unless @sort_field + return "sort(field='#{@sort_field}')" unless @sort_descending + return "sort(field='#{@sort_field}', descending='true')" + end + + # Cption can be piped + # Gets the last count items + def tail + return unless @tail + "tail(count=#{@tail})" + end + + # Cption can be piped + # Gets the first count items (rows) + def truncate + return unless @truncate + "truncate(count=#{@truncate})" + end + + # Cption can be piped + # Reverses the order of the rows + def reverse + return unless @reverse + "reverse()" + end + + # Cption can be piped + # Removes items (rows) with duplicate values in the specified field (column) + def unique + return unless @unique + "unique(field='#{@unique}')" + end + + # Cption can be piped + # Sanitizes the output for HTML-safe rendering. To sanitize all returned fields, omit the field parameter. + def sanitize + return unless @sanitize + return "sanitize()" unless @sanitize_field + "sanitize(field='#{@sanitize_field}')" + end + + # Its always advisable to order the pipe when there are more than one + # pipe commands available else unexpected results might get returned + # reorder_pipe_command {:from => 1, :to => 0} + # the values in the hash are the element numbers + # + def reorder_pipe_command(args) + return if current_pipe_command_types.empty? + if args[:from].nil? or args[:to].nil? + raise Yql::Error, "Not able to move pipe commands. Wrong element numbers. Please try again" + end + args.values.each do |element| + if element > current_pipe_command_types.size-1 + raise Yql::Error, "Not able to move pipe commands. Wrong element numbers. Please try again" + end + end + element_to_be_inserted_at = args[:from] < args[:to] ? args[:to]+1 : args[:to] + element_to_be_removed = args[:from] < args[:to] ? args[:from] : args[:from]+1 + current_pipe_command_types.insert(element_to_be_inserted_at, current_pipe_command_types.at(args[:from])) + current_pipe_command_types.delete_at(element_to_be_removed) + end + + + # Remove a command that will be piped to the yql query + def remove_pipe_command(command) + current_pipe_command_types.delete(command) + end + + private + + def pipe_commands + return if current_pipe_command_types.empty? + '| ' + current_pipe_command_types.map{|c| self.send(c)}.join(' | ') + end + + def build_select_query + [select_statement, conditions, limit, pipe_commands].compact.join(' ') + end + + def construct_query + return build_select_query unless @use + return [@use, build_select_query].join('; ') + end + + def select_statement + select = "select #{column_select} from #{table}" + return select unless per_page + with_pagination(select) + end + + def with_pagination(select) + self.current_page ||= 1 + offset = (current_page - 1) * per_page + last_record = current_page * per_page + "#{select}(#{offset},#{last_record})" + end + + def column_select + @select ? @select : '*' + end + + end + +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..46eb34f --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,3 @@ +$: << File.join(File.dirname(__FILE__), "/../lib") +require 'spec' +require 'yql' \ No newline at end of file diff --git a/spec/yql/client_spec.rb b/spec/yql/client_spec.rb new file mode 100644 index 0000000..e47aaa3 --- /dev/null +++ b/spec/yql/client_spec.rb @@ -0,0 +1,9 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +describe Yql::Client do + + before(:each) do + + end + +end \ No newline at end of file diff --git a/spec/yql/query_builder_spec.rb b/spec/yql/query_builder_spec.rb new file mode 100644 index 0000000..547946a --- /dev/null +++ b/spec/yql/query_builder_spec.rb @@ -0,0 +1,9 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +describe Yql::QueryBuilder do + + before(:each) do + + end + +end \ No newline at end of file diff --git a/yql.gemspec b/yql.gemspec new file mode 100644 index 0000000..876ece2 --- /dev/null +++ b/yql.gemspec @@ -0,0 +1,29 @@ +Gem::Specification.new do |s| + s.name = %q{yql} + s.version = "0.0.1" + + s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= + s.authors = ["Nasir Jamal"] + s.date = %q{2010-06-20} + s.description = %q{Yql is a ruby wrapper for Yahoo Query Language.} + s.email = %q{nas35_in@yahoo.com} + s.extra_rdoc_files = ["README.rdoc"] + s.files = ["History.txt", + "Manifest.txt", + "README.rdoc", + "lib/yql.rb", + "lib/yql/client.rb", + "lib/yql/error.rb", + "lib/yql/query_builder.rb", + ] + s.has_rdoc = true + s.homepage = %q{http://github.com/nas/yql} + s.rdoc_options = ["--charset=UTF-8"] + s.require_paths = ["lib"] + s.rubygems_version = %q{1.3.0} + s.summary = %q{Yql is a ruby wrapper for Yahoo Query Language.} + + s.platform = Gem::Platform::RUBY + s.required_ruby_version = '>=1.8' + +end