Permalink
Browse files

last few weeks of changes

  • Loading branch information...
2 parents af71367 + d357734 commit a71047ff8148cd443332e064ae391357d35417a6 @bentut committed Oct 13, 2012
View
@@ -90,6 +90,9 @@ def source_type
end
end
+ # instead of this "is pseudo_history. May want to do something like this periodically"
+ # might need to define a pseudohistory task somewhere
+ # DataSource.where("eval LIKE '%bls_histextend_date_format_correct.xls%'").each {|ds| ds.mark_as_pseudo_history}
def is_pseudo_history?
pseudo_history_sources = [
"/Volumes/UHEROwork/data/rawdata/History/inc_hist.xls",
@@ -100,19 +103,20 @@ def is_pseudo_history?
pseudo_history_sources.each { |phs| return true if source_eval.index(phs) }
return false
end
-
- def DataPoint.set_pseudo_history
- DataPoint.all.each do |dp|
- begin
- ph = dp.is_pseudo_history?
- dp.update_attributes(:pseudo_history => true) if ph and !dp.pseudo_history
- dp.update_attributes(:pseudo_history => false) if !ph and dp.pseudo_history
- rescue
- puts "error for dp #{dp.id}"
- end
- end
- 0
- end
+
+ #this never finishes running. Doesn't seem to catch all the stuff I want either
+ # def DataPoint.set_pseudo_history
+ # DataPoint.all.each do |dp|
+ # begin
+ # ph = dp.is_pseudo_history?
+ # dp.update_attributes(:pseudo_history => true) if ph and !dp.pseudo_history
+ # dp.update_attributes(:pseudo_history => false) if !ph and dp.pseudo_history
+ # rescue
+ # puts "error for dp #{dp.id}"
+ # end
+ # end
+ # 0
+ # end
def source_type_code
case source_type
View
@@ -160,7 +160,29 @@ def reload_source
# dp.update_attributes(:history => Time.now) if (dp.history.nil? and dates.index(dp.date_string).nil?)
# end
# end
+
+ # DataSource.where("eval LIKE '%bls_histextend_date_format_correct.xls%'").each {|ds| ds.mark_as_pseudo_history}
+
+ def mark_as_pseudo_history
+ puts "marking ds: #{self.id}"
+ data_points.each {|dp| dp.update_attributes(:pseudo_history => true) }
+ end
+
+ def mark_as_pseudo_history_before(date_string)
+ puts "marking ds: #{self.id}"
+ data_points.where("date_string < '#{date_string}'" ).each {|dp| dp.update_attributes(:pseudo_history => true) }
+ end
+ def unmark_as_pseudo_history
+ puts "unmarking ds: #{self.id}"
+ data_points.each {|dp| dp.update_attributes(:pseudo_history => false) }
+ end
+
+ def unmark_as_pseudo_history_before(date_string)
+ puts "unmarking ds: #{self.id}"
+ data_points.where("date_string < '#{date_string}'" ).each {|dp| dp.update_attributes(:pseudo_history => false) }
+ end
+
def delete_all_other_sources
s = self.series
s.data_sources_by_last_run.each {|ds| ds.delete unless ds.id == self.id}
View
@@ -352,6 +352,7 @@ def data_from_datapoints
def scaled_data_no_pseudo_history(round_to = 3)
data_hash = {}
self.units ||= 1
+ self.units = 1000 if name[0..2] == "TGB" #hack for the tax scaling. Should not save units
data_points.each do |dp|
data_hash[dp.date_string] = (dp.value / self.units).round(round_to) if dp.current and !dp.pseudo_history
end
@@ -483,6 +484,12 @@ def load_from_fred(code)
new_transformation("loaded series : #{code} from FRED website", series_data)
end
+ def days_in_period
+ series_data = {}
+ data.each {|date_string, val| series_data[date_string] = date_string.to_date.days_in_period(self.frequency) }
+ Series.new_transformation("days in time periods", series_data, Series.frequency_from_code(frequency))
+ end
+
def Series.load_from_fred(code, frequency)
series_data = DataHtmlParser.new.get_fred_series(code)
Series.new_transformation("loaded series : #{code} from FRED website", series_data, Series.frequency_from_code(frequency))
@@ -21,8 +21,7 @@
<a href="/prognoz_data_files">Prognoz Output Files</a> |
<a href="/series">Data Series</a> |
<a href="/data_source_downloads">Downloads</a> |
- <a href="/mapping">Coverage</a> |
- <a href="/investigate">Accuracy</a> |
+ <a href="/investigate_visual">Investigation Dashboard</a> |
<a href="/data_lists">Exports</a> |
Search
@@ -38,7 +38,7 @@
<span class='current-datapoint'>
<% bgcolor = DataSource.find(cdp.data_source_id).color rescue "FFF" %>
<div class='datapoint current-datapoint' style='background-color:#<%= bgcolor %>'>
- <%= "%.3f" % cdp_val %><sup><%= (Time.now.to_date - cdp.created_at.to_date).to_i %></sup>
+ <%= "%.3f" % cdp_val %><sup><%= (Time.now.to_date - cdp.created_at.to_date).to_i %><%="(ph)" if cdp.pseudo_history %></sup>
</div>
<%
#series.data_points.where(:current => false,
@@ -54,7 +54,7 @@
<% dp_val = dp.value / series.units %>
<% bgcolor = DataSource.find(dp.data_source_id).color rescue "FFF" %>
<div class='datapoint' style='background-color:#<%= bgcolor %>'>
- <%= "%.3f" % dp_val %><sup><%= (Time.now.to_date - dp.created_at.to_date).to_i %></sup>
+ <%= "%.3f" % dp_val %><sup><%= (Time.now.to_date - dp.created_at.to_date).to_i %><%="(ph)" if dp.pseudo_history %></sup>
</div>
<% end %>
</span>
@@ -55,6 +55,12 @@ def semi_s
return "#{self.year}-07-01" if [7,8,9,10,11,12].include?(self.mon)
end
+ def days_in_period(frequency)
+ return (self.leap? ? 366 : 365) if frequency == "year"
+ return self.days_in_month + (self >> 1).days_in_month + (self >> 2).days_in_month if frequency == "quarter"
+ return self.days_in_month if frequency == "month"
+ end
+
def days_in_month
Time.days_in_month(self.month, self.year)
end
@@ -29,7 +29,7 @@ def observation_at(index)
col = @col_processor.compute(index, @cached_files, handle, sheet)
# puts "#{row}, #{col}"
- #puts "trying: h:#{handle}, s:#{sheet}, r:#{row}, c:#{col}, p:#{path}"
+# puts "trying: h:#{handle}, s:#{sheet}, r:#{row}, c:#{col}, p:#{path}"
worksheet = @cached_files.xls(handle, sheet, path)
rescue RuntimeError => e
puts e.message unless @handle_processor.date_sensitive?
@@ -47,8 +47,8 @@ def observation_at(index)
raise e
end
- # puts worksheet.cell(row,col)
- observation_value = parse_cell(worksheet.cell(row,col))
+ # puts worksheet.cell(row,col)
+ observation_value = parse_cell(worksheet.cell(row,col))
# puts observation_value
return "END" if observation_value == "BREAK IN DATA" unless @handle_processor.date_sensitive?
@@ -170,6 +170,7 @@ def aremos_comparison_display_array
# end
def data_diff(comparison_data, digits_to_round)
+ self.units = 1000 if name[0..2] == "TGB" #hack for the tax scaling. Should not save units
cdp = current_data_points
diff_hash = {}
results = []
View
@@ -115,9 +115,9 @@ def mc_ma_county_share_for(county_abbrev, series_prefix = self.name.split("@")[0
start_date = "#{series_prefix}NS@#{county_abbrev}.#{f}".ts.first_value_date
end_date = "#{series_prefix}NS@#{county_abbrev}.#{f}".ts.get_last_complete_december_datestring
historical = "#{series_prefix}NS@#{county_abbrev}.#{f}".ts.moving_average_offset_early(start_date,end_date) / "#{series_prefix}NS@HI.#{f}".ts.moving_average_offset_early(start_date,end_date) * self
- historical.print
+ #historical.print
mean_corrected_historical = historical / historical.annual_sum * "#{series_prefix}NS@#{county_abbrev}.#{f}".ts.annual_sum
- mean_corrected_historical.print
+ #mean_corrected_historical.print
current_year = "#{series_prefix}NS@#{county_abbrev}.#{f}".ts.backward_looking_moving_average.get_last_incomplete_year / "#{series_prefix}NS@HI.#{f}".ts.backward_looking_moving_average.get_last_incomplete_year * self
new_transformation("Share of #{name} using ratio of #{series_prefix}NS@#{county_abbrev}.#{f} over #{series_prefix}NS@HI.#{f} using a mean corrected moving average (offset early) and a backward looking moving average for the current year",
mean_corrected_historical.data.series_merge(current_year.data))
Oops, something went wrong.

0 comments on commit a71047f

Please sign in to comment.