Permalink
Browse files

Implemented the series reloading without rake files

  • Loading branch information...
1 parent 0f86d12 commit b5297aab387fba8ab822720d937ce76e84edb547 @bentut committed Mar 1, 2013
@@ -6,4 +6,14 @@ def download_notification(update_info, update_summary)
@update_summary = update_summary
mail(:to => "bentut@gmail.com", :subject => "A spreadsheet was updated: #{update_info}")
end
+
+ def series_refresh_notification(circular_series, stale_data_sources, ds_count, errors)
+ @circular_series = circular_series
+ @stale_data_sources = stale_data_sources
+ #add this in later
+ #@empty_data_sources = empty_data_sources
+
+ @errors = errors
+ mail(:to => "bentut@gmail.com", :subject => "[Udaman Series Reload] #{ds_count} Datasources loaded : #{@errors.count} / #{@circular_series.count} / #{@stale_data_sources.count} ")
+ end
end
View
@@ -5,6 +5,12 @@ class DataSource < ActiveRecord::Base
belongs_to :series
has_many :data_points
+ composed_of :last_run,
+ :class_name => "Time",
+ :mapping => %w(last_run_in_seconds to_r),
+ :constructor => Proc.new { |t| Time.at(t) },
+ :converter => Proc.new { |t| t.is_a?(Time) ? t : Time.at(t/1000.0) }
+
#DataSource.where("eval LIKE '%load_from%'").all.each {|ds| puts "#{ds.series.name} - #{ds.eval}" }
def DataSource.type_buckets
@@ -129,11 +135,7 @@ def DataSource.set_dependencies
def create_from_form
Series.eval Series.find(self.series_id).name, self.eval
- # puts "OK!"
return true
- # rescue Exception
- # puts "PROBLEM!!"
- # return false
end
def setup
@@ -158,17 +160,6 @@ def clear_and_reload_source
self.update_attributes(:description => s.name, :last_run => Time.now, :runtime => (Time.now - t))
end
- # def mark_history
- # #puts "SOURCE HISTORY-----------------------"
- # dates = data.keys
- # #puts dates.sort
- # self.data_points.each do |dp|
- # #puts "history: #{dp.history}, date: #{dp.date_string}"
- # dp.update_attributes(:history => nil) if (!dp.history.nil? and !dates.index(dp.date_string).nil?)
- # dp.update_attributes(:history => Time.now) if (dp.history.nil? and dates.index(dp.date_string).nil?)
- # end
- # end
-
# DataSource.where("eval LIKE '%bls_histextend_date_format_correct.xls%'").each {|ds| ds.mark_as_pseudo_history}
def mark_as_pseudo_history
@@ -209,29 +200,16 @@ def current?
rescue
return false
end
-
- # def delete_no_series
- # self.data_points.each do |dp|
- # dp.delete
- # end
- # super
- # end
-
+
def delete_data_points
self.data_points.each do |dp|
dp.delete
end
end
def delete
- #series_id = self.series_id
delete_data_points
super
- #s = Series.find series_id
- # puts "Series name: #{s.name}, Sources:#{s.data_sources_by_last_run.count}"
-
- #put this in a separate function
- #s.data_sources_by_last_run.each {|ds| ds.reload_source}
end
@@ -255,13 +233,7 @@ def get_eval_statement
end
def print_eval_statement
- #might not need to do something different for seasonal adjustment anymore
- #if self.eval.index("apply_seasonal_adjustment").nil?
- #puts "\"#{self.series.name}\".ts_append_eval %Q|#{self.eval}|" if self.mode == "append"
- puts "\"#{self.series.name}\".ts_eval= %Q|#{self.eval}|" #if self.mode == "set"
- # else
- # puts self.eval
- # end
+ puts "\"#{self.series.name}\".ts_eval= %Q|#{self.eval}|"
end
def set_dependencies
View
@@ -783,21 +783,28 @@ def last_data_added_string
last_data_added.strftime("%B %e, %Y")
end
- def Series.run_all_dependencies(series_list, already_run)
+ #currently runs in 3 hrs (for all series..if concurrent series could go first, that might be nice)
+ #could do everything with no dependencies first and do all of those in concurrent fashion...
+ #to find errors, or broken series, maybe update the ds with number of data points loaded on last run?
+
+ def Series.run_all_dependencies(series_list, already_run, errors, eval_statements)
series_list.each do |s_name|
- next unless already_run.index(s_name).nil?
+ next unless already_run[s_name].nil?
s = s_name.ts
begin
- already_run = Series.run_all_dependencies(s.new_dependencies, already_run)
+ Series.run_all_dependencies(s.new_dependencies, already_run, errors, eval_statements)
rescue
puts "THIS IS THE ONE THAT BROKE"
puts s.id
puts s.name
end
- s.reload_sources
- already_run.push(s_name)
+ errors.concat s.reload_sources
+ eval_statements.concat(s.data_sources_by_last_run.map {|ds| ds.get_eval_statement})
+ puts "---------count-------------------"
+ puts errors.count
+ puts eval_statements.count
+ already_run[s_name] = true
end
- return already_run
end
def Series.smart_update(series_names_finished = [], series_to_finish = Series.all, depth = 0 )
@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+ </head>
+ <body>
+ <h3>Errors</h3>
+ <p>
+ <% @errors.each do |line| %>
+ <%= line %><br />
+ <% end %>
+ </p>
+
+ <h3>Circular Series</h3>
+ <p>
+ <% @circular_series.each do |line| %>
+ <%= line %><br />
+ <% end %>
+ </p>
+ <h3>Stale DataSources</h3>
+ <p>
+ <% @stale_data_sources.each do |ds| %>
+ <%= ds.id %> : <%= ds.eval %><br />
+ <% end %>
+ </p>
+ <h3>Empty DataSources</h3>
+ <p>implement later</p>
+
+ </body>
+</html>
@@ -0,0 +1,9 @@
+class AddLastRunMilliseconds < ActiveRecord::Migration
+ def self.up
+ add_column :data_sources, :last_run_in_seconds, :decimal, :precision => 17, :scale => 3
+ end
+
+ def self.down
+ remove_column :data_sources, :last_run_in_seconds
+ end
+end
@@ -0,0 +1,9 @@
+class DropOriginalLastRun < ActiveRecord::Migration
+ def self.up
+ remove_column :data_sources, :last_run
+ end
+
+ def self.down
+ add_column :data_sources, :last_run, :datetime
+ end
+end
View
@@ -1,3 +1,4 @@
+# encoding: UTF-8
# This file is auto-generated from the current state of the database. Instead
# of editing this file, please use the migrations feature of Active Record to
# incrementally modify your database, and then regenerate this schema definition.
@@ -10,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20121002094044) do
+ActiveRecord::Schema.define(:version => 20130301003211) do
create_table "aremos_series", :force => true do |t|
t.string "name"
@@ -83,12 +84,12 @@
t.integer "series_id"
t.string "description"
t.string "eval"
- t.datetime "last_run"
t.text "dependencies"
t.string "color"
t.float "runtime"
t.datetime "created_at"
t.datetime "updated_at"
+ t.decimal "last_run_in_seconds", :precision => 17, :scale => 3
end
add_index "data_sources", ["series_id"], :name => "index_data_sources_on_series_id"
@@ -103,7 +103,7 @@ def DownloadPreprocessor.match?(elem, spreadsheet, match_type, header_in, search
cell_value = spreadsheet[row-1][col-1].to_s if options[:sheet].nil?
cell_value = spreadsheet.cell(row,col).to_s unless options[:sheet].nil?
options[:header_name].split("[or]").each do |header|
- #puts "Searching for #{header} in row:#{row} col:#{col}"
+ #puts "Searching for #{header} in row:#{row} col:#{col} / VALUE: #{cell_value}"
result = match(cell_value, header) if match_type == :hiwi
result = match_prefix(cell_value, header) if match_type == :prefix
result = match_trim_elipsis(cell_value,header) if match_type == :trim_elipsis
@@ -5,7 +5,6 @@ def initialize(integer_pattern, cached_files = nil)
end
def compute(index, cached_files = nil, handle = nil, sheet = nil)
-
return Integer(@integer_pattern)
rescue
p = @integer_pattern.split ":"
View
@@ -124,18 +124,18 @@ def Series.print_multi_sources
end
return 1
end
-
+
def reload_sources
- begin
- puts self.name
- # self.data_sources_by_last_run.each do |ds|
- # ds.print_eval_statement
- # ds.reload_source
- # end
- return 0
- rescue Exception
- puts "SOMETHING BROKE -----------------------------------------------"
+ errors = []
+ self.data_sources_by_last_run.each do |ds|
+ begin
+ ds.reload_source
+ rescue Exception
+ errors.push("DataSource #{ds.id} for #{self.name} : #{self.id}")
+ puts "SOMETHING BROKE with source #{ds.id} in series #{self.name} (#{self.id})-----------------------------------------------"
+ end
end
+ return errors
end
def print_source_eval_statements
View
@@ -0,0 +1,61 @@
+"VISJPNS@HI.M".ts_eval= %Q|"VISJPNS@HI.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISJPNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-02-01", :sheet=>"Japan", :row=>"header_range:col:1:TOTAL VISITORS:1:100", :col=>2, :frequency=>"M" })/1000|
+"VISJPNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_FINAL@hawaiitourismauthority.org", { :file_type => "xls", :start_date => "1989-01-01", :sheet => "5", :row => "repeat:5:16", :col => "block:4:10:12", :frequency => "M" })/1000|
+"VISJPNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"Japan", :row=>"header:col:1:TOTAL VISITORS", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISJPNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"Days by MMA", :row=>"header_range:col:1:Japan:11:20", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISJPNS@HI.M".ts_eval= %Q|"VISJPNS@HI.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/Manual/visfix.xls"|
+"VISJP@HI.M".ts_eval= %Q|"VISJP@HI.M".ts.apply_seasonal_adjustment :multiplicative|
+"VISJP@HI.M".ts_eval= %Q|"VISJP@HI.M".ts.load_mean_corrected_sa_from "/Volumes/UHEROwork/data/tour/seasadj/sadata.xls"|
+"VISJPNS@HON.M".ts_eval= %Q|"VISJPNS@HON.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISJPNS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-01-01", :sheet=>"Japan", :row=>"header_range:col:1:Oahu:10:45:no_okina", :col=>2, :frequency=>"M" })/1000|
+"VISJPNS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"Japan", :row=>"header:col:1:Oahu:no_okina", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISJP@HON.M".ts_eval= %Q|"VISJP@HI.M".ts.mc_ma_county_share_for("HON")|
+"VISJP@HON.M".ts_eval= %Q|"VISJP@HI.M".ts.mc_ma_county_share_for("HON","VISJP")|
+"VISUSWNS@HI.M".ts_eval= %Q|"VISUSWNS@HI.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISUSWNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-02-01", :sheet=>"sheet_num:3", :row=>"header_range:col:1:TOTAL VISITORS:1:100", :col=>2, :frequency=>"M" })/1000|
+"VISUSWNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_FINAL@hawaiitourismauthority.org", { :file_type => "xls", :start_date => "1989-01-01", :sheet => "5", :row => "repeat:5:16", :col => "block:2:10:12", :frequency => "M" })/1000|
+"VISUSWNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"US West", :row=>"header:col:1:TOTAL VISITORS", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISUSWNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"Days by MMA", :row=>"header_range:col:1:US West:11:20", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISUSENS@HI.M".ts_eval= %Q|"VISUSENS@HI.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISUSENS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-02-01", :sheet=>"sheet_num:4", :row=>"header_range:col:1:TOTAL VISITORS:1:100", :col=>2, :frequency=>"M" })/1000|
+"VISUSENS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_FINAL@hawaiitourismauthority.org", { :file_type => "xls", :start_date => "1989-01-01", :sheet => "5", :row => "repeat:5:16", :col => "block:3:10:12", :frequency => "M" })/1000|
+"VISUSENS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"US East", :row=>"header:col:1:TOTAL VISITORS", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISUSENS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"Days by MMA", :row=>"header_range:col:1:US East:11:20", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISUSNS@HI.M".ts_eval= %Q|"VISUSNS@HI.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd3_hist.xls"|
+"VISUSNS@HI.M".ts_eval= %Q|"VISUSWNS@HI.M".ts + "VISUSENS@HI.M".ts|
+"VISUS@HI.M".ts_eval= %Q|"VISUS@HI.M".ts.apply_seasonal_adjustment :additive|
+"VISUS@HI.M".ts_eval= %Q|"VISUS@HI.M".ts.load_mean_corrected_sa_from "/Volumes/UHEROwork/data/tour/seasadj/sadata.xls"|
+"VISUSWNS@HON.M".ts_eval= %Q|"VISUSWNS@HON.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISUSWNS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-01-01", :sheet=>"sheet_num:3", :row=>"header_range:col:1:Oahu:10:45:no_okina", :col=>2, :frequency=>"M" })/1000|
+"VISUSWNS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"US West", :row=>"header:col:1:Oahu:no_okina", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISUSENS@HON.M".ts_eval= %Q|"VISUSENS@HON.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISUSENS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-01-01", :sheet=>"sheet_num:4", :row=>"header_range:col:1:Oahu:10:45:no_okina", :col=>2, :frequency=>"M" })/1000|
+"VISUSENS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"US East", :row=>"header:col:1:Oahu:no_okina", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISUSNS@HON.M".ts_eval= %Q|"VISUSNS@HON.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd3_hist.xls"|
+"VISUSNS@HON.M".ts_eval= %Q|"VISUSWNS@HON.M".ts + "VISUSENS@HON.M".ts|
+"VISUS@HON.M".ts_eval= %Q|"VISUS@HI.M".ts.mc_ma_county_share_for("HON")|
+"VISNS@HI.M".ts_eval= %Q|"VISNS@HI.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-02-01", :sheet=>"HL", :row=>"header_range:col:1:TOTAL VISITORS:1:59", :col=>2, :frequency=>"M" })/1000|
+"VISNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_FINAL@hawaiitourismauthority.org", { :file_type => "xls", :start_date => "1989-01-01", :sheet => "5", :row => "repeat:5:16", :col => "block:11:10:12", :frequency => "M" })/1000|
+"VISNS@HI.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"sheet_num:1", :row=>"header_range:col:1:TOTAL VISITORS:1:94", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISRESNS@HI.M".ts_eval= %Q|"VISNS@HI.M".ts - "VISUSNS@HI.M".ts - "VISJPNS@HI.M".ts|
+"VISRES@HI.M".ts_eval= %Q|"VISRES@HI.M".ts.load_mean_corrected_sa_from "/Volumes/UHEROwork/data/tour/seasadj/sadata.xls"|
+"VISRES@HI.M".ts_eval= %Q|"VISRES@HI.M".ts.apply_seasonal_adjustment :additive|
+"VISNS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-02-01", :sheet=>"sheet_num:8", :row=>"header_range:col:1:Oahu:24:32:no_okina", :col=>2, :frequency=>"M" })/1000|
+"VISNS@HON.M".ts_eval= %Q|"VISNS@HON.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISNS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_FINAL@hawaiitourismauthority.org", { :file_type => "xls", :start_date => "1990-01-01", :sheet => "6", :row => "repeat:18:29", :col => "block:2:3:12", :frequency => "M" })/1000|
+"VISNS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"sheet_num:1", :row=>"header_range:col:1:Oahu:1:94:no_okina", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISNS@HON.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"Days by Island", :row=>"header_range:col:1:Oahu:13:23:no_okina", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISRESNS@HON.M".ts_eval= %Q|"VISNS@HON.M".ts - "VISUSNS@HON.M".ts - "VISJPNS@HON.M".ts|
+"VISRES@HON.M".ts_eval= %Q|"VISRES@HI.M".ts.mc_ma_county_share_for("HON")|
+"VISNS@HAW.M".ts_eval= %Q|Series.load_from_download( "TOUR_%b%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2011-02-01", :sheet=>"sheet_num:8", :row=>"header_range:col:1:Big Island[or]hawaii island:24:32:no_okina", :col=>2, :frequency=>"M" })/1000|
+"VISNS@HAW.M".ts_eval= %Q|"VISNS@HAW.M".tsn.load_from "/Volumes/UHEROwork/data/rawdata/History/tour_upd1_hist.xls"|
+"VISNS@HAW.M".ts_eval= %Q|Series.load_from_download( "TOUR_FINAL@hawaiitourismauthority.org", { :file_type => "xls", :start_date => "1990-01-01", :sheet => "6", :row => "repeat:96:107", :col => "block:2:3:12", :frequency => "M" })/1000|
+"VISNS@HAW.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", {:file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"sheet_num:1", :row=>"header_range:col:1:Big Island[or]Hawaii Island:1:94", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISNS@HAW.M".ts_eval= %Q|Series.load_from_download( "TOUR_HIST%y@hawaiitourismauthority.org", { :file_type=>"xls", :start_date=>"2010-01-01", :sheet=>"Days by Island", :row=>"header_range:col:1:Hawaii Island:13:23:no_okina", :col=>"repeat:2:13", :frequency=>"M" })/1000|
+"VISDEMETRA_MC@HI.M".ts_eval= %Q|"VIS@HI.M".ts.load_mean_corrected_sa_from "/Volumes/UHEROwork/data/tour/seasadj/sadata.xls"|
+"VISDEMETRA_MC@HI.M".ts_eval= %Q| "VIS@HI.M".ts.apply_seasonal_adjustment :additive|
+"VIS@HON.M".ts_eval= %Q| "VISDEMETRA_MC@HI.M".ts.mc_ma_county_share_for("HON","VIS").trim("1990-01-01","1999-12-01")|
+"VIS@HON.M".ts_eval= %Q| "VISDEMETRA_MC@HI.M".ts.mc_ma_county_share_for("HON","VIS").trim|
+"VIS@HON.M".ts_eval= %Q|"VISJP@HON.M".ts + "VISUS@HON.M".ts + "VISRES@HON.M".ts|
+"VIS@HON.A".ts_eval= %Q|"VIS@HON.M".ts.aggregate(:year, :sum)|
Oops, something went wrong.

0 comments on commit b5297aa

Please sign in to comment.