Permalink
Browse files

Merge remote-tracking branch 'origin/push-service-feedback-to-pp'

  • Loading branch information...
2 parents 84430d5 + 3785d23 commit d81da902904997ec4a64d89d2cfdf98a9fe62c01 @alext alext committed Jan 28, 2014
View
@@ -30,6 +30,12 @@ gem "sidekiq", "2.17.1"
gem "statsd-ruby", "1.2.1", require: "statsd"
gem 'logstasher', '0.4.1'
gem 'jquery-tablesorter', '1.8.1'
+gem 'whenever', '0.9.0', require: false
+if ENV['API_DEV']
+ gem "gds-api-adapters", :path => '../gds-api-adapters'
+else
+ gem "gds-api-adapters", "8.3.0"
+end
group :development do
gem "quiet_assets", "1.0.2"
View
@@ -2,6 +2,7 @@ GEM
remote: https://rubygems.org/
remote: https://BnrJb6FZyzspBboNJzYZ@gem.fury.io/govuk/
specs:
+ PriorityQueue (0.1.2)
actionmailer (3.2.16)
actionpack (= 3.2.16)
mail (~> 2.5.4)
@@ -49,6 +50,7 @@ GEM
timers (~> 1.1.0)
childprocess (0.3.5)
ffi (~> 1.0, >= 1.0.6)
+ chronic (0.10.2)
commonjs (0.2.6)
connection_pool (1.2.0)
crack (0.3.2)
@@ -80,6 +82,12 @@ GEM
actionpack (>= 3.0)
formtastic-bootstrap (2.1.1)
formtastic (~> 2.2)
+ gds-api-adapters (8.3.0)
+ link_header
+ lrucache (~> 0.1.1)
+ null_logger
+ plek
+ rest-client (~> 1.6.3)
gds-sso (9.1.0)
omniauth-gds (>= 3.0.0)
rack-accept (~> 0.4.4)
@@ -119,9 +127,12 @@ GEM
libv8 (3.16.14.3)
libwebsocket (0.1.5)
addressable
+ link_header (0.0.8)
logstash-event (1.1.5)
logstasher (0.4.1)
logstash-event (~> 1.1.0)
+ lrucache (0.1.4)
+ PriorityQueue (~> 0.1.2)
mail (2.5.4)
mime-types (~> 1.16)
treetop (~> 1.4.8)
@@ -208,6 +219,8 @@ GEM
redis-store (1.1.4)
redis (>= 2.2)
ref (1.0.5)
+ rest-client (1.6.7)
+ mime-types (>= 1.16)
rubyzip (0.9.9)
selenium-webdriver (2.25.0)
childprocess (>= 0.2.5)
@@ -257,6 +270,9 @@ GEM
webmock (1.9.0)
addressable (>= 2.2.7)
crack (>= 0.1.7)
+ whenever (0.9.0)
+ activesupport (>= 2.3.4)
+ chronic (>= 0.6.3)
xml-simple (1.1.1)
xpath (0.1.4)
nokogiri (~> 1.3)
@@ -279,6 +295,7 @@ DEPENDENCIES
cucumber-rails (= 1.3.0)
exception_notification (= 3.0.1)
formtastic-bootstrap (= 2.1.1)
+ gds-api-adapters (= 8.3.0)
gds-sso (= 9.1.0)
gds_zendesk (= 1.0.1)
jquery-rails
@@ -301,3 +318,4 @@ DEPENDENCIES
unicorn (= 4.3.1)
validates_timeliness (= 3.0.14)
webmock (= 1.9.0)
+ whenever (= 0.9.0)
@@ -0,0 +1,27 @@
+require 'date'
+require 'gds_api/performance_platform/data_in'
+require 'support/requests/anonymous/service_feedback_aggregated_metrics'
+
+class ServiceFeedbackPPUploaderWorker
+ include Sidekiq::Worker
+ include Support::Requests::Anonymous
+
+ def perform(year, month, day, transaction_slug)
+ Rails.logger.info("Uploading statistics for #{year}-#{month}-#{day}, slug #{transaction_slug}")
+ api = GdsApi::PerformancePlatform::DataIn.new(
+ PP_DATA_IN_API[:url],
+ bearer_token: PP_DATA_IN_API[:bearer_token]
+ )
+ request_details = ServiceFeedbackAggregatedMetrics.new(Date.new(year, month, day), transaction_slug).to_h
+ api.submit_service_feedback_day_aggregate(transaction_slug, request_details)
+ end
+
+ def self.run
+ yesterday = Date.yesterday
+ slugs = Support::Requests::Anonymous::ServiceFeedback.transaction_slugs
+ slugs.each do |transaction_slug|
+ perform_async(yesterday.year, yesterday.month, yesterday.day, transaction_slug)
+ end
+ Rails.logger.info("Queued upload for #{slugs.size} slugs")
+ end
+end
@@ -0,0 +1,4 @@
+PP_DATA_IN_API = {
+ url: "http://www.performance.dev.gov.uk",
+ bearer_token: 'XXXXXXXXXXXXX'
+}
View
@@ -0,0 +1,11 @@
+# default cron env is "/usr/bin:/bin" which is not sufficient as govuk_env is in /usr/local/bin
+env :PATH, '/usr/local/bin:/usr/bin:/bin'
+
+set :output, {:error => 'log/cron.error.log', :standard => 'log/cron.log'}
+
+# We need Rake to use our own environment
+job_type :rake, "cd :path && govuk_setenv support bundle exec rake :task :output"
+
+every 1.day, :at => '12:30 am' do
+ rake "push_service_feedback_to_pp"
+end
View
@@ -0,0 +1,18 @@
+require 'redis'
+
+class RedisClient
+ include Singleton
+
+ attr_reader :connection
+
+ def initialize
+ @connection = Redis.new(config.symbolize_keys)
+ end
+
+private
+
+ def config
+ YAML.load_file(Rails.root.join("config", "redis.yml"))
+ end
+
+end
@@ -10,6 +10,21 @@ class ServiceFeedback < AnonymousContact
validates :details, length: { maximum: 2 ** 16 }
validates_inclusion_of :service_satisfaction_rating, in: (1..5).to_a
validates :url, url: true, length: { maximum: 2048 }, allow_nil: true
+
+ def self.transaction_slugs
+ uniq.pluck(:slug).sort
+ end
+
+ def self.aggregates_by_rating
+ zero_defaults = Hash[*(1..5).map {|n| [n, 0] }.flatten]
+ select("service_satisfaction_rating, count(*) as cnt").
+ group(:service_satisfaction_rating).
+ inject(zero_defaults) { |memo, result| memo[result[:service_satisfaction_rating]] = result[:cnt]; memo }
+ end
+
+ def self.with_comments_count
+ where("details IS NOT NULL").count
+ end
end
end
end
@@ -0,0 +1,54 @@
+require 'date'
+require_relative "service_feedback"
+
+module Support
+ module Requests
+ module Anonymous
+ class ServiceFeedbackAggregatedMetrics
+ def initialize(day, slug)
+ @day = day
+ @slug = slug
+ end
+
+ def to_h
+ metadata.merge(aggregates)
+ end
+
+ private
+ def aggregates
+ by_rating = filter_by_day_and_slug.aggregates_by_rating
+
+ results_array = by_rating.map {|rating, count| ["rating_#{rating}", count] }.flatten +
+ [ "comments", filter_by_day_and_slug.with_comments_count ] +
+ [ "total", by_rating.values.inject(:+)]
+
+ Hash[*results_array]
+ end
+
+ def metadata
+ {
+ "_id" => metric_id,
+ "_timestamp" => start_timestamp,
+ "period" => "day"
+ }
+ end
+
+ def metric_id
+ "#{@day.strftime("%Y%m%d")}_#{@slug}"
+ end
+
+ def start_timestamp
+ @day.to_time.iso8601
+ end
+
+ def filter_by_day_and_slug
+ ServiceFeedback.where(slug: @slug, created_at: time_interval)
+ end
+
+ def time_interval
+ (@day.to_time...@day.to_time + 1.day)
+ end
+ end
+ end
+ end
+end
@@ -0,0 +1,12 @@
+desc "Trigger an upload of service feedback data to the performance platform"
+task :push_service_feedback_to_pp => :environment do
+ require File.join(Rails.root, 'app', 'workers', 'service_feedback_pp_uploader_worker')
+ require 'volatile_lock'
+
+ if VolatileLock.new('support:push_service_feedback_to_pp').obtained?
+ ServiceFeedbackPPUploaderWorker.run
+ puts "ServiceFeedbackPPUploaderWorker invoked"
+ else
+ puts "ServiceFeedbackPPUploaderWorker: skipping, couldn't obtain lock (probably the task has already run on another node)"
+ end
+end
View
@@ -0,0 +1,44 @@
+require 'redis_client'
+
+class VolatileLock
+ class FailedToSetExpiration < StandardError; end
+
+ # expiration_time takes care of time-drifts on our
+ # servers. defaults to 10.minutes assuming our servers
+ # won't realistically have a greater time-drift.
+ def initialize(key, expiration_time = 10.minutes)
+ @key = key
+ @expiration_time = expiration_time
+ end
+
+ def obtained?
+ delete_possibly_stale_keys
+
+ result = redis.setnx(@key, hostname)
+ result = expire if result
+ result
+ end
+
+private
+
+ def expire
+ result = redis.expire(@key, @expiration_time)
+ return true if result
+
+ redis.del(@key)
+ raise FailedToSetExpiration
+ end
+
+ def delete_possibly_stale_keys
+ redis.del(@key) if redis.get(@key) == hostname
+ end
+
+ def redis
+ RedisClient.instance.connection
+ end
+
+ def hostname
+ Socket.gethostname
+ end
+
+end
No changes.
@@ -0,0 +1,61 @@
+require 'test_helper'
+require 'volatile_lock'
+
+class VolatileLockTest < ActiveSupport::TestCase
+
+ def teardown
+ redis.del('foo', 'bar')
+ end
+
+ def redis
+ RedisClient.instance.connection
+ end
+
+ def volatile_lock(key, expiration_time = 1.second)
+ VolatileLock.new(key, expiration_time)
+ end
+
+ test "starts by deleting possibly stale locks created by the same host" do
+ redis.set('foo', Socket.gethostname)
+ assert volatile_lock('foo').obtained?
+ end
+
+ test "ensures only one lock is obtained per key across hosts" do
+ Socket.stubs(:gethostname).returns('pluto')
+ assert volatile_lock('foo').obtained?
+
+ Socket.stubs(:gethostname).returns('mars')
+ refute volatile_lock('foo').obtained?
+ end
+
+ test "allows multiple locks to be obtained if keys differ" do
+ assert volatile_lock('foo').obtained?
+ assert volatile_lock('bar').obtained?
+ end
+
+ test "allows expiration_time to be changed" do
+ redis = mock(get: nil, setnx: true)
+ redis.expects(:expire).with('foo', 30.seconds).returns(true)
+ VolatileLock.any_instance.stubs(:redis).returns(redis)
+
+ volatile_lock('foo', 30.seconds).obtained?
+ end
+
+ context "failing to set expiration time" do
+ should "raise FailedToSetExpiration" do
+ redis = mock(get: nil, setnx: true, del: true, expire: false)
+ VolatileLock.any_instance.stubs(:redis).returns(redis)
+
+ assert_raises(VolatileLock::FailedToSetExpiration) { volatile_lock('foo').obtained? }
+ end
+
+ should "delete the persisted key" do
+ redis = mock(get: nil, setnx: true, expire: false)
+ redis.expects(:del).with('foo')
+ VolatileLock.any_instance.stubs(:redis).returns(redis)
+
+ volatile_lock('foo').obtained? rescue VolatileLock::FailedToSetExpiration
+ end
+ end
+
+end
@@ -0,0 +1,50 @@
+require 'test_helper'
+
+module Support
+ module Requests
+ module Anonymous
+ class ServiceFeedbackAggregatedMetricsTest < Test::Unit::TestCase
+ def setup
+ create_feedback(rating: 1, slug: "abcde", created_at: Date.new(2013,2,10))
+ create_feedback(rating: 3, slug: "apply_carers_allowance", created_at: Date.new(2013,2,10))
+ create_feedback(rating: 2, details: "abcde", slug: "apply_carers_allowance", created_at: Date.new(2013,2,10))
+ @stats = ServiceFeedbackAggregatedMetrics.new(Date.new(2013,2,10), "apply_carers_allowance").to_h
+ end
+
+ def create_feedback(options)
+ f = ServiceFeedback.create!(slug: options[:slug] || "a",
+ details: options[:details],
+ service_satisfaction_rating: options[:rating],
+ javascript_enabled: true)
+ f.update_attribute(:created_at, options[:created_at])
+ end
+
+ context "metadata" do
+ should "generate an id based on the slug and date" do
+ assert_equal "20130210_apply_carers_allowance", @stats["_id"]
+ end
+
+ should "set the period to a day" do
+ assert_equal "day", @stats["period"]
+ end
+
+ should "set the start time correctly" do
+ assert_equal "2013-02-10T00:00:00+00:00", @stats["_timestamp"]
+ end
+ end
+
+ context "aggregated metrics" do
+ should "include rating summaries" do
+ assert_equal 0, @stats["rating_1"]
+ assert_equal 1, @stats["rating_2"]
+ assert_equal 1, @stats["rating_3"]
+ assert_equal 0, @stats["rating_4"]
+ assert_equal 0, @stats["rating_5"]
+ assert_equal 2, @stats["total"]
+ assert_equal 1, @stats["comments"]
+ end
+ end
+ end
+ end
+ end
+end
Oops, something went wrong. Retry.

0 comments on commit d81da90

Please sign in to comment.