Permalink
Browse files

fixed some scheduling

  • Loading branch information...
bentut committed Mar 1, 2013
1 parent b5297aa commit 32546c7671c35faeef78e43d3f49acb3a8988cda
Showing with 102 additions and 142 deletions.
  1. +1 −50 app/models/series.rb
  2. +75 −72 config/schedule.rb
  3. +7 −0 lib/tasks/WriteDash.rake
  4. +19 −20 lib/tasks/source_map.rake
View
@@ -806,56 +806,7 @@ def Series.run_all_dependencies(series_list, already_run, errors, eval_statement
already_run[s_name] = true
end
end
-
- def Series.smart_update(series_names_finished = [], series_to_finish = Series.all, depth = 0 )
- return series_to_finish if series_to_finish.count == 0 or depth == 25
- series_to_finish_new = []
- series_to_finish.each do |series|
- if series.open_dependencies(series_names_finished).count == 0
- series_names_finished.push series.name
- series.reload_sources
- else
- series_to_finish_new.push series
- end
- end
- puts "# Series To finish: #{series_to_finish_new.count}"
- #puts series_names_finished.count
- puts "# -----end------"
- puts "\n\n\n\n\n\n\n\n\n\n\n\n\n"
- if (series_to_finish.count == series_to_finish_new.count)
- # series_to_finish.each do |stuck|
- # puts stuck.name
- # end
- return series_to_finish_new
- end
- return Series.smart_update(series_names_finished, series_to_finish_new, depth+1)
- end
-
- def Series.output_database_rebuild_statements(series_names_finished = [], series_to_finish = Series.all, depth = 0 )
- return series_to_finish if series_to_finish.count == 0 or depth == 25
- series_to_finish_new = []
- series_to_finish.each do |series|
- if series.open_dependencies(series_names_finished).count == 0
- series_names_finished.push series.name
- #puts "# #{series.name}"
- series.print_source_eval_statements
- else
- series_to_finish_new.push series
- end
- end
- puts "# Series To finish: #{series_to_finish_new.count}"
- #puts series_names_finished.count
- puts "# -----end------"
- puts "\n\n\n\n\n\n\n\n\n\n\n\n\n"
- if (series_to_finish.count == series_to_finish_new.count)
- # series_to_finish.each do |stuck|
- # puts stuck.name
- # end
- return series_to_finish_new
- end
- return Series.output_database_rebuild_statements(series_names_finished, series_to_finish_new, depth+1)
- end
-
+
def Series.missing_from_aremos
name_buckets = {}
(AremosSeries.all_names - Series.all_names).each {|name| name_buckets[name[0]] ||= []; name_buckets[name[0]].push(name)}
View
@@ -32,97 +32,100 @@
rake "reload_aremos"
end
-every 1.day, :at => "12:00 am" do
- rake "daily_history_load"
-end
+# every 1.day, :at => "12:00 am" do
+# rake "daily_history_load"
+# end
every 1.day, :at => "#{hour}:00 am" do
rake "update_seats_links"
rake "update_vis_history_links"
end
-every 1.day, :at => "#{hour}:00 am" do
- rake "jp_upd_a"
- rake "jp_upd_q"
- rake "jp_upd_m"
-end
-
every 1.day, :at => "#{hour}:10 am" do
- rake "tour_PC_upd"
- rake "tour_seats_upd"
- rake "tour_upd"
- rake "tour_ocup_upd"
+ rake "reload_all_series"
end
+# every 1.day, :at => "#{hour}:00 am" do
+# rake "jp_upd_a"
+# rake "jp_upd_q"
+# rake "jp_upd_m"
+# end
+#
+# every 1.day, :at => "#{hour}:10 am" do
+# rake "tour_PC_upd"
+# rake "tour_seats_upd"
+# rake "tour_upd"
+# rake "tour_ocup_upd"
+# end
every 1.day, :at => "#{bls_hour}:00 am" do
rake "update_bea_links"
end
-every 1.day, :at => "#{bls_hour}:05 am" do
- rake "gsp_upd"
- rake "inc_upd_q"
- rake "inc_upd_a"
- rake "com_upd"
-end
-
-every 1.day, :at => "#{bls_hour}:30 am" do
- rake "us_upd_a"
- rake "us_upd_q"
- rake "us_upd_m"
-end
-
-every 1.day, :at => "#{bls_hour.to_i+1}:15 am" do
- rake "bls_cpi_upd_m"
- rake "bls_job_upd_m"
- rake "bls_cpi_upd_s"
- rake "hiwi_upd"
-end
-
-every 1.day, :at => "#{bls_hour.to_i+1}:30 am" do
- rake "bls_identities"
- rake "bea_identities"
-end
+# every 1.day, :at => "#{bls_hour}:05 am" do
+# rake "gsp_upd"
+# rake "inc_upd_q"
+# rake "inc_upd_a"
+# rake "com_upd"
+# end
+#
+# every 1.day, :at => "#{bls_hour}:30 am" do
+# rake "us_upd_a"
+# rake "us_upd_q"
+# rake "us_upd_m"
+# end
+#
+# every 1.day, :at => "#{bls_hour.to_i+1}:15 am" do
+# rake "bls_cpi_upd_m"
+# rake "bls_job_upd_m"
+# rake "bls_cpi_upd_s"
+# rake "hiwi_upd"
+# end
+#
+# every 1.day, :at => "#{bls_hour.to_i+1}:30 am" do
+# rake "bls_identities"
+# rake "bea_identities"
+# end
every 1.day, :at => "#{bls_hour.to_i+2}:30 am" do
rake "write_ur_dash"
end
-every 1.day, :at => "#{hour}:30 am" do
- rake "uic_upd"
- rake "const_upd_q"
- rake "const_upd_m"
-end
-
-every 1.day, :at => "#{hour}:35 am" do
- rake "tax_upd"
-end
-
-every 1.day, :at => "#{hour}:40 am" do
- rake "tour_rev_upd"
-end
-
-every 1.day, :at => "#{bls_hour.to_i+2}:00 am" do
- rake "expenditures_and_nbi"
- rake "visitor_identities"
- rake "const_identities"
-end
-
-every 1.day, :at => "#{hour.to_i+1}:30 am" do
- rake "tax_identities"
-end
-
-every 1.day, :at => "#{bls_hour.to_i+2}:15 am" do
- rake "run_aggregations"
- rake "run_aggregations2"
- rake "run_aggregations3"
- rake "run_aggregations4"
- rake "run_aggregations5"
-end
-
-every 1.day, :at => "#{bls_hour.to_i+2}:30 am" do
- rake "bls_nbis"
-end
+# every 1.day, :at => "#{hour}:30 am" do
+# rake "uic_upd"
+# rake "const_upd_q"
+# rake "const_upd_m"
+# end
+#
+# every 1.day, :at => "#{hour}:35 am" do
+# rake "tax_upd"
+# end
+#
+# every 1.day, :at => "#{hour}:40 am" do
+# rake "tour_rev_upd"
+# end
+#
+# every 1.day, :at => "#{bls_hour.to_i+2}:00 am" do
+# rake "expenditures_and_nbi"
+# rake "visitor_identities"
+# rake "const_identities"
+# end
+#
+# every 1.day, :at => "#{hour.to_i+1}:30 am" do
+# rake "tax_identities"
+# end
+#
+# every 1.day, :at => "#{bls_hour.to_i+2}:15 am" do
+# rake "run_aggregations"
+# rake "run_aggregations2"
+# rake "run_aggregations3"
+# rake "run_aggregations4"
+# rake "run_aggregations5"
+# end
+#
+# every 1.day, :at => "#{bls_hour.to_i+2}:30 am" do
+# rake "bls_nbis"
+# end
every 1.day, :at => "#{hour.to_i+5}:30 am" do
rake "gen_investigate_csv"
View
@@ -64,6 +64,13 @@ end
task :write_ur_dash => :environment do
t = Time.now
+
+ #now just updating these on the fly
+ series_to_refresh = ["UR@HI.M", "UR@HON.M", "UR@HAW.M", "UR@KAU.M", "UR@MAU.M"]
+ eval_statements = []
+ errors = []
+ Series.run_all_dependencies(series_to_refresh, {}, errors, eval_statements)
+
post_name = "Unemployment Rates in Hawaii (Seasonally Adjusted)"
post_address ='http://www.uhero.hawaii.edu/123'
new_data_series = {}
View
@@ -30,6 +30,25 @@ task :reload_aremos => :environment do
end
+task :reload_all_series => :environment do
+ t = Time.now
+ circular = Series.find_first_order_circular
+ CSV.open("public/rake_time.csv", "a") {|csv| csv << ["circular reference check", "%.2f" % (Time.now - t) , t.to_s, Time.now.to_s] }
+
+ t = Time.now
+ series_to_refresh = Series.all_names - circular.uniq
+ #series_to_refresh = ["VIS@HON.A"]
+ eval_statements = []
+ errors = []
+ Series.run_all_dependencies(series_to_refresh, {}, errors, eval_statements)
+ CSV.open("public/rake_time.csv", "a") {|csv| csv << ["complete series reload", "%.2f" % (Time.now - t) , t.to_s, Time.now.to_s] }
+ File.open('lib/tasks/REBUILD.rb', 'w') {|file| eval_statements.each {|line| file.puts(line)} }
+
+ #719527 is 1970-01-01 in mysql days (actually one day earlier because of HST = -8 GMT)
+ inactive_ds = DataSource.where("FROM_DAYS(719527 + TRUNCATE(last_run_in_seconds,0) / 3600 / 24) < FROM_DAYS(TO_DAYS(NOW()))").order(:last_run_in_seconds)
+
+ DataLoadMailer.series_refresh_notification(circular, inactive_ds, eval_statements.count, errors).deliver
+end
task :daily_history_load => :environment do
t = Time.now
@@ -109,25 +128,5 @@ task :daily_history_load => :environment do
# ---------------------------------------------------------
end
-task :reload_all_series => :environment do
- t = Time.now
- circular = Series.find_first_order_circular
- CSV.open("public/rake_time.csv", "a") {|csv| csv << ["circular reference check", "%.2f" % (Time.now - t) , t.to_s, Time.now.to_s] }
-
- t = Time.now
- #series_to_refresh = Series.all_names - circular.uniq
- series_to_refresh = ["VIS@HON.A"]
- eval_statements = []
- errors = []
- Series.run_all_dependencies(series_to_refresh, {}, errors, eval_statements)
- CSV.open("public/rake_time.csv", "a") {|csv| csv << ["complete series reload", "%.2f" % (Time.now - t) , t.to_s, Time.now.to_s] }
- File.open('lib/tasks/REBUILD.rb', 'w') {|file| eval_statements.each {|line| file.puts(line)} }
-
- #719527 is 1970-01-01 in mysql days (actually one day earlier because of HST = -8 GMT)
- inactive_ds = DataSource.where("FROM_DAYS(719527 + TRUNCATE(last_run_in_seconds,0) / 3600 / 24) < FROM_DAYS(TO_DAYS(NOW()))").order(:last_run_in_seconds)
-
- DataLoadMailer.series_refresh_notification(circular, inactive_ds, eval_statements.count, errors).deliver
-
-end

0 comments on commit 32546c7

Please sign in to comment.