Permalink
Browse files

merge map_reduce

  • Loading branch information...
2 parents a4a3cb4 + cf61de2 commit 9c89d4f8a2d76bd1d95cf136978390fee6978c1b @lpsBetty committed Jan 21, 2012
Showing with 106 additions and 35 deletions.
  1. +3 −4 Gemfile
  2. +8 −13 Gemfile.lock
  3. +45 −10 app/models/value/map.rb
  4. +33 −4 app/models/value/prop.rb
  5. +17 −4 app/models/value/value.rb
View
@@ -1,11 +1,10 @@
source 'http://rubygems.org'
gem 'rails', '3.1.3'
-gem 'mongoid'
-gem 'bson_ext'
-gem 'json'
+gem 'mongoid', '2.3.4'
+gem 'bson_ext', '1.5.1'
+gem 'json', '1.6.3'
gem 'parallel', '0.5.11'
-gem 'mongoid-mapreduce'
gem 'jquery-rails'
gem 'typus', git: "git://github.com/fesplugas/typus.git", branch: "master"
View
21 Gemfile.lock 100755 → 100644
@@ -1,6 +1,6 @@
GIT
remote: git://github.com/fesplugas/typus.git
- revision: 4afe9a9b325f0702f2ddb074f1ebb9aabc6900ad
+ revision: 56e06de21cdad65feff2853281c4690d377573a9
branch: master
specs:
typus (3.1.5)
@@ -45,8 +45,7 @@ GEM
arel (2.2.1)
bcrypt-ruby (3.0.1)
bson (1.5.2)
- bson_ext (1.5.2)
- bson (= 1.5.2)
+ bson_ext (1.5.1)
builder (3.0.0)
coffee-rails (3.1.1)
coffee-script (>= 2.2.0)
@@ -65,7 +64,7 @@ GEM
database_cleaner (0.6.7)
diff-lcs (1.1.3)
erubis (2.7.0)
- execjs (1.2.13)
+ execjs (1.3.0)
multi_json (~> 1.0)
gherkin (2.6.9)
json (>= 1.4.6)
@@ -74,7 +73,7 @@ GEM
jquery-rails (1.0.19)
railties (~> 3.0)
thor (~> 0.14)
- json (1.6.5)
+ json (1.6.3)
kaminari (0.12.4)
rails (>= 3.0.0)
linecache19 (0.5.12)
@@ -86,13 +85,10 @@ GEM
mime-types (1.17.2)
mongo (1.5.2)
bson (= 1.5.2)
- mongoid (2.4.1)
+ mongoid (2.3.4)
activemodel (~> 3.1)
mongo (~> 1.3)
tzinfo (~> 0.3.22)
- mongoid-mapreduce (0.1.4)
- bson_ext (~> 1.3)
- mongoid (~> 2.0)
mongoid-rspec (1.4.4)
mongoid (~> 2.0)
rspec (~> 2)
@@ -177,14 +173,13 @@ PLATFORMS
ruby
DEPENDENCIES
- bson_ext
+ bson_ext (= 1.5.1)
coffee-rails (~> 3.1.1)
cucumber (= 1.1.1)
database_cleaner (= 0.6.7)
jquery-rails
- json
- mongoid
- mongoid-mapreduce
+ json (= 1.6.3)
+ mongoid (= 2.3.4)
mongoid-rspec
parallel (= 0.5.11)
rails (= 3.1.3)
@@ -19,27 +19,28 @@ def self.mapval_all params
end
def self.mapvalaggr_var params
- values = get_values(params)
- hash = values.asc(:number).group_by(&:point_id)
-
data = {}
data[params[:variable]] = []
- hash.each do |k,v|
- data[params[:variable]] << get_aggr(params[:function], v)
+ # map, reduce_avg, reduece_min, reduce_max functions are below and private
+ result = Value.collection.map_reduce(map, send("reduce_#{params[:function]}"), out: "results", query: get_query(params))
+ result.find().each do |hash|
+ data[params[:variable]] << hash["value"]
end
+
output_hash("val", params, data)
end
def self.mapvalaggr_all params
data = {}
Variable.all.each do |var|
data[var.name] = []
- values = get_values(params, var.id)
- hash = values.asc(:number).group_by(&:point_id)
- hash.each do |k,v|
- data[var.name] << get_aggr(params[:function], v)
+
+ result = Value.collection.map_reduce(map, send("reduce_#{params[:function]}"), out: "results", query: get_query(params, var.id))
+ result.find().each do |hash|
+ data[var.name] << hash["value"]
end
+
end
output_hash("val", params, data)
end
@@ -136,6 +137,40 @@ def self.mapdiffaggr_all params
end
output_hash("diff", params, data)
end
-
+
+
+ private
+
+ def self.map
+ "function() {emit(this.point_id, this.number);}"
+ end
+
+ def self.reduce_avg
+ "function(key, values) {
+ var sum = 0;
+ values.forEach(function(number) {
+ sum += number;
+ });
+ return sum / values.length;
+ }"
+ end
+ def self.reduce_min
+ "function(key, values) {
+ var min = values[0];
+ values.forEach(function(number) {
+ if (number < min) min = number;
+ });
+ return min;
+ }"
+ end
+ def self.reduce_max
+ "function(key, values) {
+ var max = values[0];
+ values.forEach(function(number) {
+ if (number > max) max = number;
+ });
+ return max;
+ }"
+ end
end
@@ -3,12 +3,14 @@ class Prop < Value
# properties methods
def self.propval params
- values = get_values(params).asc(:number)
+ result = Value.collection.map_reduce(map, reduce, out: "results", query: get_query(params))
+ result = result.find().first["value"]
+
data = {}
data[params[:variable]] = {}
- data[params[:variable]]["min"] = values.first.number
- data[params[:variable]]["max"] = values.last.number
- data[params[:variable]]["avg"] = avg(values)
+ data[params[:variable]]["min"] = result["min"]
+ data[params[:variable]]["max"] = result["max"]
+ data[params[:variable]]["avg"] = result["avg"]
output_hash("val", params, data, "prop")
end
@@ -37,5 +39,32 @@ def self.propdiffaggr params
output_hash("diff", params, data, "prop")
end
+
+
+ private
+
+ def self.map
+ "function() {emit(this.scenario_id, this.number);}"
+ end
+
+ def self.reduce
+ "function(point, values) {
+ var n = {avg: 0, min: 0, max: 0};
+ var sum = 0;
+ var count = 0;
+ var min = values[0];
+ var max = values[0];
+ values.forEach(function(number) {
+ if (number < min) min = number;
+ if (number > max) max = number;
+ sum += number;
+ count += 1;
+ });
+ n.min = min;
+ n.max = max;
+ n.avg = sum / count;
+ return n;
+ }"
+ end
end
@@ -1,6 +1,5 @@
class Value
include Mongoid::Document
- include Mongoid::MapReduce
field :year, type: Integer
field :month, type: Integer
@@ -19,7 +18,7 @@ class Value
belongs_to :variable, index: true
belongs_to :point, index: true
- private
+ private
# if there is an var_id passed then we want that id, if not the params[:variable] is used
# if there is a number (no) passed then we have params[:year1] and params[:year2] (used for diff values)
@@ -32,11 +31,25 @@ def self.get_values params, var_id=false, no=""
month = params["month#{no}"]
v = Value.only(:number, :point_id)
- v = v.where(year: year) if year # for /propval/Mo/Sc/all/all/Var.Out
- v = v.where(month: month) if month # for aggregated values
+ v = v.where(year: year.to_i) if year # for /propval/Mo/Sc/all/all/Var.Out
+ v = v.where(month: month.to_i) if month # for aggregated values
v.where(model_id: model_id, scenario_id: scenario_id, variable_id: var_id)
end
+ def self.get_query params, var_id=false, no=""
+ model_id = Model.find_id_by_name(params[:model])
+ scenario_id = Scenario.find_id_by_name(params[:scenario])
+ var_id = Variable.find_id_by_name(params[:variable]) if params[:variable]
+
+ year = params["year#{no}"]
+ month = params["month#{no}"]
+
+ q = {model_id: model_id, scenario_id: scenario_id, variable_id: var_id}
+ q.merge!({year: year.to_i}) if year
+ q.merge!({month: month.to_i}) if month
+ q
+ end
+
def self.avg(array)
array.collect(&:number).sum.to_f/array.length
end

0 comments on commit 9c89d4f

Please sign in to comment.