Browse files

initial commit with comprehensive select operations

  • Loading branch information...
0 parents commit 61974f46d3485686ab496f1a4f5479081c1e5d13 nas committed Jun 20, 2010
Showing with 446 additions and 0 deletions.
  1. +4 −0 History.txt
  2. +7 −0 Manifest.txt
  3. +104 −0 README.rdoc
  4. +11 −0 lib/yql.rb
  5. +65 −0 lib/yql/client.rb
  6. +19 −0 lib/yql/error.rb
  7. +186 −0 lib/yql/query_builder.rb
  8. +3 −0 spec/spec_helper.rb
  9. +9 −0 spec/yql/client_spec.rb
  10. +9 −0 spec/yql/query_builder_spec.rb
  11. +29 −0 yql.gemspec
4 History.txt
@@ -0,0 +1,4 @@
+=== 0.1.1 2010-06-20
+
+* 1 major enhancement:
+ * Initial release
7 Manifest.txt
@@ -0,0 +1,7 @@
+History.txt
+Manifest.txt
+README.rdoc
+lib/yql.rb
+lib/yql/client.rb
+lib/yql/error.rb
+lib/yql/query_builder.rb
104 README.rdoc
@@ -0,0 +1,104 @@
+==DESCRIPTION
+
+A basic Ruby Wrapper for interacting programatically with YQL API.
+
+I started working on this library during ScienceHackDay held at Guardian, London on 19/06/2010-20/06/2010
+
+==TODO
+
+1. Add Unit Tests
+2. Add oauth
+3. Add table creation
+4. Add update / insert / delete operations
+
+==INSTALLATION
+
+sudo gem source --add http://rubygems.org
+
+sudo gem install yql
+
+
+==USAGE
+
+require 'rubygems'
+
+require 'yql'
+
+===Building Query
+
+
+====Finders
+
+yql = Yql::Client.new
+
+query = Yql::QueryBuilder.new 'yelp.review.search'
+
+query.to_s #=> "select * from yelp.review.search"
+
+query.find #=> "select * from yelp.review.search limit 1"
+
+query.limit = 4
+
+query.to_s #=> "select * from yelp.review.search limit 4"
+
+query.find_all #=> "select * from yelp.review.search"
+
+
+====Conditions
+
+query.conditions = "term like '%pizza%'"
+
+query.to_s #=> "select * from yelp.review.search where term='%pizza%'"
+
+query.conditions = {:term => 'pizza'}
+
+query.to_s #=> "select * from yelp.review.search where term = 'pizza'"
+
+query.conditions = {:term => 'pizza', :location => 'london', 'ywsid' => '6L0Lc-yn1OKMkCKeXLD4lg'}
+
+query.to_s #=> "select * from yelp.review.search where term='pizza' and location='london' and ywsid= '6L0Lc-yn1OKMkCKeXLD4lg'"
+
+query.select = 'user_photo_url, state'
+
+yql.query = query
+result = yql.get
+
+yql.format = 'json'
+result = yql.get
+
+
+===Piped Filters
+
+query.unique = 'name'
+
+query.to_s #=> "select title, Rating, LastReviewIntro from yelp.review.search where ywsid='6L0Lc-yn1OKMkCKeXLD4lg' and term='pizza' and location='london' | unique(field='name')"
+
+query.tail = 4
+
+query.to_s #=> "select title, Rating, LastReviewIntro from yelp.review.search where ywsid='6L0Lc-yn1OKMkCKeXLD4lg' and term='pizza' and location='london' | unique(field='name') | tail(count=4)"
+
+query.reorder_pipe_command :from => 1, :to => 0
+
+query.to_s #=> "select title, Rating, LastReviewIntro from yelp.review.search where ywsid='6L0Lc-yn1OKMkCKeXLD4lg' and term='pizza' and location='london' | tail(count=4) | unique(field='name')"
+
+yql.format = 'json'
+result = yql.get
+
+
+====Pagination
+
+query.per_page = 10
+query.current_page = 1
+
+yql.query = query
+result = yql.get
+
+
+===Describe and show tables
+
+query = Yql::QueryBuilder.describe_table('yelp.review.search')
+
+query = Yql::QueryBuilder.show_tables
+
+yql.query = query
+result = yql.get
11 lib/yql.rb
@@ -0,0 +1,11 @@
+$:.unshift(File.dirname(__FILE__)) unless
+ $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'rubygems'
+require 'CGI'
+require 'net/http'
+require 'net/https'
+require 'rexml/document'
+require 'yql/error.rb'
+require 'yql/client.rb'
+require 'yql/query_builder.rb'
65 lib/yql/client.rb
@@ -0,0 +1,65 @@
+require 'net/http'
+module Yql
+
+ class Client
+
+ BASE_URL = 'query.yahooapis.com'
+ VERSION = 'v1'
+ URL_SUFFIX = 'public/yql'
+ YQL_ENV = 'http://datatables.org/alltables.env'
+
+ attr_accessor :query, :diagnostics, :format
+
+ def initialize(args={})
+ @diagnostics = args[:diagnostics] #true or false
+ @version = args[:version]
+ @query = args[:query]
+ @format = args[:format] || 'xml'
+ end
+
+ def query
+ @query.kind_of?(Yql::QueryBuilder) ? @query.to_s : @query
+ end
+
+ def version
+ @version ||= VERSION
+ end
+
+ def full_url
+ "#{version}/#{URL_SUFFIX}"
+ end
+
+ def get
+ if query.nil?
+ raise Yql::IncompleteRequestParameter, "Query not specified"
+ end
+ http = Net::HTTP.new(BASE_URL, Net::HTTP.https_default_port)
+ http.use_ssl = true
+ path = "/#{version}/#{URL_SUFFIX}"
+ result = http.post(path, parameters)
+ #raise(Yql::ResponseFailure, result.response) unless result.code == '200'
+ return result.body unless format == 'xml'
+ REXML::Document.new(result.body)
+ end
+
+ def parameters
+ url_parameters = "q=#{CGI.escape(query)}&env=#{YQL_ENV}"
+ url_parameters = add_format(url_parameters)
+ add_diagnostics(url_parameters)
+ end
+
+ def add_format(existing_parameters)
+ return unless existing_parameters
+ return existing_parameters unless format
+ return existing_parameters + "&format=#{format}"
+ end
+
+ def add_diagnostics(existing_parameters)
+ return unless existing_parameters
+ return existing_parameters unless diagnostics
+ return existing_parameters + "&diagnostics=true"
+ end
+
+ end
+
+end
19 lib/yql/error.rb
@@ -0,0 +1,19 @@
+module Yql
+ class Error < StandardError
+
+ def initialize(data)
+ @data = data
+ super
+ end
+ end
+
+ class ResponseFailure < Error
+ end
+
+ class NoResult < Error
+ end
+
+ class IncompleteRequestParameter < Error
+ end
+
+end
186 lib/yql/query_builder.rb
@@ -0,0 +1,186 @@
+module Yql
+
+ class QueryBuilder
+
+ attr_accessor :table, :conditions, :limit, :truncate, :sanitize_field, :select,
+ :sort_field, :current_pipe_command_types, :sort_descending,
+ :per_page, :current_page
+
+ def initialize(table, args = {})
+ @select = args[:select]
+ @table = table
+ @use_statement = args[:use]
+ @conditions = args[:conditions]
+ @limit = args[:limit]
+ @tail = args[:tail]
+ @reverse = args[:reverse]
+ @unique = args[:unique]
+ @sanitize = args[:sanitize]
+ @sort_descending = args[:sort_descending]
+ @sanitize_field = args[:sanitize_field]
+ @sort_field = args[:sort_field]
+ @current_pipe_command_types = []
+ @per_page = args[:per_page]
+ end
+
+ def find
+ self.limit = 1
+ "#{construct_query}"
+ end
+
+ def find_all
+ self.limit = nil
+ construct_query
+ end
+
+ def self.show_tables
+ "show tables;"
+ end
+
+ def self.describe_table(table)
+ "desc #{table};"
+ end
+
+ def describe_table
+ Yql::QueryBuilder.describle_table(table)
+ end
+
+ def to_s
+ construct_query
+ end
+
+ def limit
+ return unless @limit
+ "limit #{@limit}"
+ end
+
+ # Conditions can either be provided as hash or plane string
+ def conditions
+ if @conditions.kind_of?(String)
+ cond = @conditions
+ elsif @conditions.kind_of?(Hash)
+ cond = @conditions.collect do |k,v|
+ val = v.kind_of?(String) ? "'#{v}'" : v
+ "#{k.to_s}=#{val}"
+ end
+ cond = cond.join(' and ')
+ else
+ return
+ end
+ return "where #{cond}"
+ end
+
+ %w{sort tail truncate reverse unique sanitize}.each do |method|
+ self.send(:define_method, "#{method}=") do |param|
+ instance_variable_set("@#{method}", param)
+ current_pipe_command_types << method unless current_pipe_command_types.include?(method)
+ end
+ end
+
+ # Cption can be piped
+ # Sorts the result set according to the specified field (column) in the result set.
+ def sort
+ return unless @sort_field
+ return "sort(field='#{@sort_field}')" unless @sort_descending
+ return "sort(field='#{@sort_field}', descending='true')"
+ end
+
+ # Cption can be piped
+ # Gets the last count items
+ def tail
+ return unless @tail
+ "tail(count=#{@tail})"
+ end
+
+ # Cption can be piped
+ # Gets the first count items (rows)
+ def truncate
+ return unless @truncate
+ "truncate(count=#{@truncate})"
+ end
+
+ # Cption can be piped
+ # Reverses the order of the rows
+ def reverse
+ return unless @reverse
+ "reverse()"
+ end
+
+ # Cption can be piped
+ # Removes items (rows) with duplicate values in the specified field (column)
+ def unique
+ return unless @unique
+ "unique(field='#{@unique}')"
+ end
+
+ # Cption can be piped
+ # Sanitizes the output for HTML-safe rendering. To sanitize all returned fields, omit the field parameter.
+ def sanitize
+ return unless @sanitize
+ return "sanitize()" unless @sanitize_field
+ "sanitize(field='#{@sanitize_field}')"
+ end
+
+ # Its always advisable to order the pipe when there are more than one
+ # pipe commands available else unexpected results might get returned
+ # reorder_pipe_command {:from => 1, :to => 0}
+ # the values in the hash are the element numbers
+ #
+ def reorder_pipe_command(args)
+ return if current_pipe_command_types.empty?
+ if args[:from].nil? or args[:to].nil?
+ raise Yql::Error, "Not able to move pipe commands. Wrong element numbers. Please try again"
+ end
+ args.values.each do |element|
+ if element > current_pipe_command_types.size-1
+ raise Yql::Error, "Not able to move pipe commands. Wrong element numbers. Please try again"
+ end
+ end
+ element_to_be_inserted_at = args[:from] < args[:to] ? args[:to]+1 : args[:to]
+ element_to_be_removed = args[:from] < args[:to] ? args[:from] : args[:from]+1
+ current_pipe_command_types.insert(element_to_be_inserted_at, current_pipe_command_types.at(args[:from]))
+ current_pipe_command_types.delete_at(element_to_be_removed)
+ end
+
+
+ # Remove a command that will be piped to the yql query
+ def remove_pipe_command(command)
+ current_pipe_command_types.delete(command)
+ end
+
+ private
+
+ def pipe_commands
+ return if current_pipe_command_types.empty?
+ '| ' + current_pipe_command_types.map{|c| self.send(c)}.join(' | ')
+ end
+
+ def build_select_query
+ [select_statement, conditions, limit, pipe_commands].compact.join(' ')
+ end
+
+ def construct_query
+ return build_select_query unless @use
+ return [@use, build_select_query].join('; ')
+ end
+
+ def select_statement
+ select = "select #{column_select} from #{table}"
+ return select unless per_page
+ with_pagination(select)
+ end
+
+ def with_pagination(select)
+ self.current_page ||= 1
+ offset = (current_page - 1) * per_page
+ last_record = current_page * per_page
+ "#{select}(#{offset},#{last_record})"
+ end
+
+ def column_select
+ @select ? @select : '*'
+ end
+
+ end
+
+end
3 spec/spec_helper.rb
@@ -0,0 +1,3 @@
+$: << File.join(File.dirname(__FILE__), "/../lib")
+require 'spec'
+require 'yql'
9 spec/yql/client_spec.rb
@@ -0,0 +1,9 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Yql::Client do
+
+ before(:each) do
+
+ end
+
+end
9 spec/yql/query_builder_spec.rb
@@ -0,0 +1,9 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Yql::QueryBuilder do
+
+ before(:each) do
+
+ end
+
+end
29 yql.gemspec
@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+ s.name = %q{yql}
+ s.version = "0.0.1"
+
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+ s.authors = ["Nasir Jamal"]
+ s.date = %q{2010-06-20}
+ s.description = %q{Yql is a ruby wrapper for Yahoo Query Language.}
+ s.email = %q{nas35_in@yahoo.com}
+ s.extra_rdoc_files = ["README.rdoc"]
+ s.files = ["History.txt",
+ "Manifest.txt",
+ "README.rdoc",
+ "lib/yql.rb",
+ "lib/yql/client.rb",
+ "lib/yql/error.rb",
+ "lib/yql/query_builder.rb",
+ ]
+ s.has_rdoc = true
+ s.homepage = %q{http://github.com/nas/yql}
+ s.rdoc_options = ["--charset=UTF-8"]
+ s.require_paths = ["lib"]
+ s.rubygems_version = %q{1.3.0}
+ s.summary = %q{Yql is a ruby wrapper for Yahoo Query Language.}
+
+ s.platform = Gem::Platform::RUBY
+ s.required_ruby_version = '>=1.8'
+
+end

0 comments on commit 61974f4

Please sign in to comment.