Merge branch 'feature/historics' into develop

README.md

@@ -4,7 +4,7 @@ DataSift
 The official Ruby library for accessing the DataSift API. See
 http://datasift.net for full details and to sign up for an account.
 
-The examples and tests use the username and API key in config.yml.
+The examples use the username and API key in config.yml unless otherwise noted.
 
 Install Instructions
 --------------------

examples/historics.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+ruby historics/$3.rb $1 $2 ${@:4}

examples/historics/create-from-csdl.rb

@@ -0,0 +1,71 @@
+# This script creates a new Historics query from a file containing CSDL.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Function to display usage instructions with an optional error message.
+def usage(message = '', end_of_story = true)
+	puts message + '\n' unless message.length() == 0
+	puts
+	puts 'Usage: create-from-csdl <username> <api_key> \\'
+	puts '            <csdl_filename> <start> <end> <sources> <name> <sample>'
+	puts
+	puts 'Where: csdl_filename = the stream hash the query should run'
+	puts '       start         = the start date for the query (YYYYMMDDHHMMSS)'
+	puts '       end           = the end date for the query (YYYYMMDDHHMMSS)'
+	puts '       sources       = comma separated list of data sources (e.g. twitter)'
+	puts '       name          = a friendly name for the query'
+	puts '       sample        = the sample rate'
+	puts
+	puts 'Example'
+	puts '       create-from-csdl csdl.txt 20120101000000 20120101235959 \\'
+	puts '                      twitter HistoricsQuery123 100'
+	puts
+	exit 1 unless not end_of_story
+end
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Check that we have enough arguments
+usage() unless env.args.size() == 6
+
+# Read the arguments
+csdl_filename = env.args[0]
+start_date    = env.args[1]
+end_date      = env.args[2]
+sources       = env.args[3].split(',')
+name          = env.args[4]
+sample        = env.args[5]
+
+# Parse the dates
+start_date = DateTime.strptime(start_date, '%Y%m%d%H%M%S')
+end_date   = DateTime.strptime(end_date,   '%Y%m%d%H%M%S')
+
+# Read the CSDL
+csdl = File.open(csdl_filename, 'r').read
+
+begin
+	# Create the definition
+	definition = env.user.createDefinition(csdl)
+
+	# Create the Historics query
+	historic = definition.createHistoric(start_date, end_date, sources, name, sample)
+
+	# Prepare the query
+	historic.prepare()
+
+	# Display the details
+	env.displayHistoricDetails(historic)
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+ rescue DataSift::DataSiftError => err
+ 	puts 'ERR: ' + err
+end

examples/historics/create-from-hash.rb

@@ -0,0 +1,65 @@
+# This script creates a new Historics query from a stream hash.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Function to display usage instructions with an optional error message.
+def usage(message = '', end_of_story = true)
+	puts message + '\n' unless message.length() == 0
+	puts
+	puts 'Usage: create-from-hash \\'
+	puts '            <username> <api_key> <hash> <start> <end> <sources> <name> <sample>'
+	puts
+	puts 'Where: hash    = the stream hash the query should run'
+	puts '       start   = the start date for the query (YYYYMMDDHHMMSS)'
+	puts '       end     = the end date for the query (YYYYMMDDHHMMSS)'
+	puts '       sources = comma separated list of data sources (e.g. twitter)'
+	puts '       name    = a friendly name for the query'
+	puts '       sample  = the sample rate'
+	puts
+	puts 'Example'
+	puts '       create-from-hash <hash> 20120101000000 20120101235959 \\'
+	puts '                      twitter HistoricsQuery123 100'
+	puts
+	exit 1 unless not end_of_story
+end
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Check that we have enough arguments
+usage() unless env.args.size() == 6
+
+# Read the arguments
+stream_hash = env.args[0]
+start_date  = env.args[1]
+end_date    = env.args[2]
+sources     = env.args[3].split(',')
+name        = env.args[4]
+sample      = env.args[5]
+
+# Parse the dates
+start_date = DateTime.strptime(start_date, '%Y%m%d%H%M%S')
+end_date   = DateTime.strptime(end_date,   '%Y%m%d%H%M%S')
+
+begin
+	# Create the Historics query
+	historic = env.user.createHistoric(stream_hash, start_date, end_date, sources, name, sample)
+
+	# Prepare the query
+	historic.prepare()
+
+	# Display the details
+	env.displayHistoricDetails(historic)
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+ rescue DataSift::DataSiftError => err
+ 	puts 'ERR: ' + err
+end

examples/historics/delete.rb

@@ -0,0 +1,30 @@
+# This script deletes Historics queries from your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Make sure we have something to do
+abort('Please specify one or more playback IDs') unless env.args.size() > 0
+
+begin
+	for playback_id in env.args
+		historic = env.user.getHistoric(playback_id)
+		print 'Deleting ' + playback_id + ', "' + historic.name + '"...'
+		historic.delete()
+		puts 'done'
+	end
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: ' + err.inspect
+end

examples/historics/env.rb

@@ -0,0 +1,37 @@
+# This class is used by the Historics examples to remove the noise of
+# dealing with command line arguments.
+#
+
+# Include the DataSift library
+require File.dirname(__FILE__) + '/../../lib/datasift'
+
+class Env
+	attr_reader :user, :args
+
+	def initialize(args = false)
+		if args === false
+			args = ARGV
+		end
+
+		abort('Please specify your DataSift username and API key as the first two command line arguments!') unless args.size() >= 2
+
+		username = args.shift
+		api_key = args.shift
+		@user = DataSift::User.new(username, api_key)
+
+		@args = args
+	end
+
+	def displayHistoricDetails(historic)
+		puts 'Playback ID: ' + historic.hash
+		puts 'Stream hash: ' + historic.stream_hash
+		puts 'Name:        ' + historic.name
+		puts 'Start time:  ' + historic.start_date.strftime('%Y-%m-%d %H:%M:%S')
+		puts 'End time:    ' + historic.end_date.strftime('%Y-%m-%d %H:%M:%S')
+		puts 'Sources:     ' + historic.sources.join(', ')
+		puts 'Sample:      ' + String(historic.sample)
+		puts 'Created at:  ' + (historic.created_at.nil? ? 'None' : historic.created_at.strftime('%Y-%m-%d %H:%M:%S'))
+		puts 'Status:      ' + historic.status
+		puts '--'
+	end
+end

examples/historics/list.rb

@@ -0,0 +1,30 @@
+# This script lists Historics queries in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+begin
+	historics = env.user.listHistorics()
+
+	if historics['historics'].size() == 0
+		puts 'No Historics queries exist in your account.'
+	else
+		for historic in historics['historics']
+			env.displayHistoricDetails(historic)
+		end
+	end
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: ' + err.inspect
+end

examples/historics/start.rb

@@ -0,0 +1,30 @@
+# This script starts Historics queries in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Make sure we have something to do
+abort('Please specify one or more playback IDs') unless env.args.size() > 0
+
+begin
+	for playback_id in env.args
+		historic = env.user.getHistoric(playback_id)
+		print 'Starting ' + playback_id + ', "' + historic.name + '"...'
+		historic.start()
+		puts 'done'
+	end
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: ' + err.inspect
+end

examples/historics/stop.rb

@@ -0,0 +1,30 @@
+# This script stops Historics queries in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Make sure we have something to do
+abort('Please specify one or more playback IDs') unless env.args.size() > 0
+
+begin
+	for playback_id in env.args
+		historic = env.user.getHistoric(playback_id)
+		print 'Stopping ' + playback_id + ', "' + historic.name + '"...'
+		historic.stop()
+		puts 'done'
+	end
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: ' + err.inspect
+end

examples/historics/view.rb

@@ -0,0 +1,28 @@
+# This script views Historics queries in your account.
+#
+# NB: Most of the error handling (exception catching) has been removed for
+# the sake of simplicity. Nearly everything in this library may throw
+# exceptions, and production code should catch them. See the documentation
+# for full details.
+#
+
+# Include the shared Env class
+require File.dirname(__FILE__) + '/env'
+
+# Create the env object. This reads the command line arguments, creates the
+# user object, and provides access to both along with helper functions.
+env = Env.new()
+
+# Make sure we have something to do
+abort('Please specify one or more playback IDs') unless env.args.size() > 0
+
+begin
+	for playback_id in env.args
+		historic = env.user.getHistoric(playback_id)
+		env.displayHistoricDetails(historic)
+	end
+
+	puts 'Rate limit remainining: ' + String(env.user.rate_limit_remaining)
+rescue DataSift::DataSiftError => err
+	puts 'ERR: ' + err.inspect
+end

lib/DataSift/definition.rb

@@ -107,7 +107,7 @@ def compile()
 				when 400
 					raise CompileFailedError, err
 				else
-					raise CompileFailedError, 'Unexpected APIError code: ' + err.http_code.to_s + ' [' + err.inspect + ']'
+					raise APIError('Unexpected APIError code: ' + err.http_code.to_s + ' [' + err.inspect + ']', err.http_code)
 				end
 			end
 		end
@@ -149,6 +149,19 @@ def getBuffered(count = false, from_id = false)
 			retval['stream']
 		end
 
+		# Create a Historics query based on this Definition.
+		# === Parameters
+		#
+		# * +start_date+ - The start date for a new Historics query.
+		# * +end_date+ - The end date for a new Historics query.
+		# * +sources+ - An array of sources for a new Historics query.
+		# * +name+ - The name for a new Historics query.
+		# * +sample+ - The sample rate for the new Historics query.
+		#
+		def createHistoric(start_date, end_date, sources, name, sample = Historic::DEFAULT_SAMPLE)
+			return Historic.new(@user, hash, start_date, end_date, sources, name, sample)
+		end
+
 		# Returns a StreamConsumer-derived object for this definition, for the
 		# given type.
 		# === Parameters

lib/DataSift/exceptions.rb

@@ -1,15 +1,17 @@
 module DataSift
-	class AccessDeniedError < StandardError; end
-	class CompileFailedError < StandardError; end
-	class InvalidDataError < StandardError; end
-	class NotYetImplementedError < StandardError; end
-	class RateLimitExceededError < StandardError; end
-	class StreamError < StandardError; end
+	class DataSiftError < StandardError; end
 
-	class APIError < StandardError
+	class AccessDeniedError < DataSiftError; end
+	class CompileFailedError < DataSiftError; end
+	class InvalidDataError < DataSiftError; end
+	class NotYetImplementedError < DataSiftError; end
+	class RateLimitExceededError < DataSiftError; end
+	class StreamError < DataSiftError; end
+
+	class APIError < DataSiftError
 		attr_reader :http_code
 
-		def initialize(http_code)
+		def initialize(http_code = -1)
 			@http_code = http_code
 		end
 	end