Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
...
Checking mergeability… Don't worry, you can still create the pull request.
  • 17 commits
  • 9 files changed
  • 0 commit comments
  • 1 contributor
View
16 app/models/route.rb
@@ -23,7 +23,14 @@ class Route < ActiveRecord::Base
# This model is part of the transport data that is versioned by data generations.
# This means they have a default scope of models valid in the current data generation.
# See lib/fixmytransport/data_generations
- exists_in_data_generation( :auto_update_fields => [:cached_description, :cached_slug] )
+ exists_in_data_generation( :temporary_identity_fields => [:id],
+ :auto_update_fields => [:cached_description,
+ :cached_slug,
+ :lat,
+ :lon,
+ :cached_area,
+ :cached_short_name,
+ :coords] )
has_many :route_sub_routes
has_many :sub_routes, :through => :route_sub_routes
has_many :route_operators, :dependent => :destroy, :uniq => true
@@ -92,7 +99,8 @@ def stop_area_codes
memoize :stop_area_codes
def transport_mode_name
- transport_mode.name
+ transport_mode.name if transport_mode
+ nil
end
def cache_area
@@ -417,8 +425,8 @@ def self.full_find(id, scope)
# Return routes with this number and transport mode that have a stop or stop area in common with
# the route given
def self.find_all_by_number_and_common_stop(new_route, options={})
- # If this is the first call to these methods on the new_route model (they are memoized), note
- # that is will be executed in the current scope of the models concerned. i.e. whatever the data
+ # If this is the first call to these methods on the new_route model (they are memoized), note
+ # that is will be executed in the current scope of the models concerned. i.e. whatever the data
# generation conditions for the current scope on the models are, will be applied.
stop_codes = new_route.stop_codes
stop_area_codes = new_route.stop_area_codes
View
2  app/models/stop.rb
@@ -61,7 +61,7 @@ class Stop < ActiveRecord::Base
:deletion_value => 'del',
:temporary_identity_fields => [:other_code],
:temp_to_perm => { :other_code => :atco_code },
- :auto_update_fields => [:cached_description, :cached_slug] )
+ :auto_update_fields => [:cached_description, :cached_slug, :metro_stop] )
has_many :stop_area_memberships
has_many :stop_areas, :through => :stop_area_memberships
validates_presence_of :common_name
View
2  config/general.yml-example
@@ -66,7 +66,7 @@ NAPTAN_DIR: '/my/naptan/dir'
NPTDR_DIR: '/my/nptdr/dir'
NPTG_DIR: '/my/nptg/dir'
NOC_DIR: '/my/noc/dir'
-NPTDR_DERIVED_DIR: '/my/nptdr-derived/dir'
+TNDS_DIR: '/my/tnds/dir'
# Incoming email
View
6 lib/fixmytransport/data_generations.rb
@@ -174,7 +174,11 @@ def get_identity_hash
end
def identity_hash
- make_id_hash(self.class.data_generation_options_hash[:identity_fields])
+ if !self.class.data_generation_options_hash[:identity_fields]
+ return {}
+ else
+ make_id_hash(self.class.data_generation_options_hash[:identity_fields])
+ end
end
def temporary_identity_hash
View
68 lib/tasks/data_loader.rb
@@ -143,13 +143,18 @@ def reference_string(model_class, instance, fields)
end
# Apply a set of changes extracted from replayable updates to a model instance
- def apply_changes(model_name, instance, changes, dryrun, verbose)
+ def apply_changes(model_name, instance, changes, dryrun, verbose, change_list)
changes.each do |attribute, values|
from_value, to_value = values
current_value = instance.send(attribute)
if from_value == current_value
- puts "#{model_name} (#{instance.id}) updating #{attribute} from #{from_value} to #{to_value}"
+ puts "#{model_name} (#{instance.id}) updating #{attribute} from #{from_value} to #{to_value}" if verbose
instance.send("#{attribute}=", to_value)
+ change_list << { :event => :update,
+ :model => instance,
+ :attribute => attribute,
+ :from_value => from_value,
+ :to_value => to_value }
elsif to_value == current_value
puts ["Discarding change of #{attribute} from #{from_value} -> #{to_value} as obsolete",
"(currently #{current_value})"].join(" ") if verbose
@@ -161,6 +166,7 @@ def apply_changes(model_name, instance, changes, dryrun, verbose)
instance.save!
end
end
+ return change_list
end
# Create a search hash that will find a model with permanent identity fields whose
@@ -223,6 +229,7 @@ def add_changes_to_migration(migration_paths, changes)
# Replay the significant updates that have been made locally to instances of a model class
# versioned with papertrail.
def replay_updates(model_class, dryrun=true, verbose=false)
+ change_list = []
if ! model_class.respond_to?(:replayable)
puts "This model is not versioned in data generations. Updates cannot be replayed."
exit(0)
@@ -235,7 +242,7 @@ def replay_updates(model_class, dryrun=true, verbose=false)
# attribute on models should be false
previous_replayable_value = model_class.replayable
model_class.replayable = false
- update_hash = get_updates(model_class, only_replayable=true)
+ update_hash = get_updates(model_class, only_replayable=true, date=nil, verbose=verbose)
model_name = model_class.to_s.downcase
update_hash.each do |identity, changes|
identity_type = identity[:identity_type]
@@ -248,7 +255,7 @@ def replay_updates(model_class, dryrun=true, verbose=false)
puts "Can't find current #{model_name} to update for #{identity_hash.inspect}" if verbose
if changes.first[:event] == 'create'
# This was a locally created object, so find the model in the previous generation
- puts "#{model_name} created locally. Looking in previous generation."
+ puts "#{model_name} created locally. Looking in previous generation." if verbose
model_class.in_generation(PREVIOUS_GENERATION) do
existing = model_class.find(:first, :conditions => identity_hash)
end
@@ -283,6 +290,7 @@ def replay_updates(model_class, dryrun=true, verbose=false)
add_changes_to_migration(migration_paths, changes)
when 'destroy'
puts "Destroying #{model_name} #{existing.id}"
+ change_list << { :event => :destroy, :model => existing }
if ! dryrun
existing.destroy
end
@@ -301,9 +309,10 @@ def replay_updates(model_class, dryrun=true, verbose=false)
migration_paths.delete(attribute)
end
end
- apply_changes(model_name, existing, migration_paths, dryrun, verbose)
+ change_list = apply_changes(model_name, existing, migration_paths, dryrun, verbose, change_list)
end
model_class.replayable = previous_replayable_value
+ return change_list
end
# Update a hash, removing keys, and values specified
@@ -332,7 +341,8 @@ def remove_ignored(diff, keys_to_ignore, values_to_ignore)
# :date - datetime the change was made on
# :event - create|update|destroy
# :changes - the significant changes in the form { attribute => [old_value, new_value] }
- def get_changes(version, model_class, only_replayable, options)
+ # The options param passed should be the data_generation_options_hash of the model class
+ def get_changes(version, model_class, only_replayable, options, verbose)
info_hash = {}
model_name = model_class.to_s.downcase
table_name = model_class.to_s.tableize
@@ -363,7 +373,7 @@ def get_changes(version, model_class, only_replayable, options)
version_model = model_class.find(version.item_id)
rescue ActiveRecord::RecordNotFound => e
puts ["New #{model_name} with id #{version.item_id} created in version #{version.id}",
- "but no further history or current #{model_name} exists"].join(" ")
+ "but no further history or current #{model_name} exists"].join(" ") if verbose
return nil
end
end
@@ -381,7 +391,7 @@ def get_changes(version, model_class, only_replayable, options)
next_version = model_class.find(version.item_id)
rescue ActiveRecord::RecordNotFound => e
puts ["#{model_name} with id #{version.item_id} updated in version #{version.id}",
- "but no further history or current #{model_name} exists"].join(" ")
+ "but no further history or current #{model_name} exists"].join(" ") if verbose
return nil
end
end
@@ -392,6 +402,7 @@ def get_changes(version, model_class, only_replayable, options)
when 'destroy'
version_model = version.reify()
info_hash[:identity] = version_model.get_identity_hash()
+ details_hash.update( :changes => {} )
info_hash[:details] = details_hash
else
raise "Unknown version event for version id #{version.id}: #{version.event}"
@@ -412,7 +423,7 @@ def get_changes(version, model_class, only_replayable, options)
# :date - datetime the change was made on
# :event - create|update|destroy
# :changes - the significant changes in the form { :attribute => [old_value, new_value] }
- def get_updates(model_class, only_replayable=true,date=nil)
+ def get_updates(model_class, only_replayable=true,date=nil, verbose=false)
update_hash = {}
options = model_class.data_generation_options_hash
condition_string = "item_type = ? "
@@ -422,16 +433,17 @@ def get_updates(model_class, only_replayable=true,date=nil)
params << date
end
if only_replayable
- condition_string += " AND replayable ='t'"
+ condition_string += " AND replayable = ?"
+ params << true
end
-
+ conditions = [condition_string] + params
# get the list of changes for this model, assemble the hash structure, only adding
# versions where some significant value has changed.
- updates = Version.find(:all, :conditions => [condition_string] + params,
+ updates = Version.find(:all, :conditions => conditions,
:order => 'created_at asc')
updates.each do |version|
- info_hash = get_changes(version, model_class, only_replayable, options)
- if info_hash && !info_hash[:details][:changes].empty?
+ info_hash = get_changes(version, model_class, only_replayable, options, verbose)
+ if info_hash && (info_hash[:details][:event] == 'destroy' || !info_hash[:details][:changes].empty?)
if update_hash[info_hash[:identity]].nil?
update_hash[info_hash[:identity]] = []
end
@@ -441,6 +453,32 @@ def get_updates(model_class, only_replayable=true,date=nil)
return update_hash
end
+ # Mark as unreplayable local updates marked as replayable for a model class that refer to an
+ # instance that does not exist and is not referred to by subsequent versions or don't contain
+ # changes to any significant fields.
+ def mark_unreplayable(model_class, dryrun, verbose)
+
+ options = model_class.data_generation_options_hash
+ condition_string = "item_type = ? AND replayable = ?"
+ params = [model_class.to_s, true]
+ conditions = [condition_string] + params
+
+ # get the list of changes for this model, assemble the hash structure, only adding
+ # versions where some significant value has changed.
+ updates = Version.find(:all, :conditions => conditions,
+ :order => 'created_at asc')
+ updates.each do |version|
+ info_hash = get_changes(version, model_class, only_replayable='t', options, verbose)
+ if info_hash.nil? || (info_hash[:details][:changes].empty? && !(info_hash[:details][:event] == 'destroy'))
+ puts "Marking #{version.id} #{version.inspect} as unreplayable" if verbose
+ version.replayable = false
+ if ! dryrun
+ version.save
+ end
+ end
+ end
+ end
+
# Load a new model instance into a generation, checking for existing record in previous generation
# and updating that or creating a new record in the new generation as appropriate
@@ -486,7 +524,7 @@ def load_instances_in_generation(model_class, parser, &block)
yield instance
end
-
+
instance.generation_low = generation
instance.generation_high = generation
# Can we find a record in the previous generation with few enough differences that we can update it
View
11 lib/tasks/temp.rake
@@ -122,6 +122,17 @@ namespace :temp do
SET replayable = 't'
WHERE item_type = 'Operator'
AND replayable is null;")
+
+ Version.connection.execute("UPDATE versions
+ SET replayable = 'f'
+ WHERE item_type = 'Route'
+ AND ((date_trunc('hour', created_at) <= '2011-04-05 18:00:00')
+ OR (date_trunc('day', created_at) = '2011-04-05'
+ AND date_trunc('hour', created_at) >= '2011-04-05 17:05:00'))")
+ Version.connection.execute("UPDATE versions
+ SET replayable = 't'
+ WHERE item_type = 'Route'
+ AND replayable is null")
end
end
View
12 lib/tasks/tnds.rake
@@ -41,7 +41,7 @@ namespace :tnds do
namespace :preload do
- desc 'Loads data from a file produced by tnds:preload:unmatched_operators and loads missing
+ desc 'Loads data from a file produced by tnds:preload:list_unmatched_operators and loads missing
operator codes and operators into the database. Accepts a file as FILE=file.
Verbose flag set by VERBOSE=1. Runs in dryrun mode unless DRYRUN=0 is specified'
task :load_unmatched_operators => :environment do
@@ -179,6 +179,8 @@ namespace :tnds do
operator = Operator.new( :short_name => operator_info[:short_name])
if !operator_info[:trading_name].blank?
operator.name = operator_info[:trading_name]
+ else
+ operator.name = operator_info[:short_name]
end
if !operator_info[:license_name].blank?
operator.vosa_license_name = operator_info[:license_name]
@@ -194,6 +196,12 @@ namespace :tnds do
# puts "#{region_name} #{operator_code}"
end
end
+ if !operator.valid?
+ puts "ERROR: Operator is invalid:"
+ puts operator.inspect
+ puts operator.errors.full_messages.join("\n")
+ exit(1)
+ end
if !dryrun
operator.save!
end
@@ -211,7 +219,7 @@ namespace :tnds do
To re-load routes from files that have already been loaded in this data generation,
supply SKIP_LOADED=0. Otherwise these files will be ignored.
Specify FIND_REGION_BY=directory if regions need to be inferred from directories.'
- task :unmatched_operators => :environment do
+ task :list_unmatched_operators => :environment do
check_for_dir
verbose = check_verbose
skip_loaded = true
View
101 lib/tasks/update.rake
@@ -8,17 +8,27 @@ namespace :update do
# Load NPTG data
Rake::Task['update:nptg'].execute
-
+
# LOAD NAPTAN DATA
Rake::Task['update:naptan'].execute
+ # Replay updates to stops, stop areas
+ ENV['MODEL'] = 'Stop'
+ Rake::Task['update:replay_updates']
+
+ ENV['MODEL'] = 'StopArea'
+ Rake::Task['update:replay_updates']
+
# LOAD NOC DATA
Rake::Task['update:noc'].execute
- # LOAD TNDS DATA
+ ENV['MODEL'] = 'Operator'
+ Rake::Task['update:replay_updates']
+ # LOAD TNDS DATA
+ Rake::Task['update:tnds']
# Rake::Task['naptan:post_load:mark_metro_stops'].execute
-
+
end
desc "Create a new data generation."
@@ -115,15 +125,27 @@ namespace :update do
Rake::Task['noc:update:operator_contacts'].execute
end
+ desc 'Update TNDS data to the current generation. Runs in dryrun mode unless DRYRUN=0
+ is specified. Verbose flag set by VERBOSE=1'
+ task :tnds => :environment do
+ ENV['DIR'] = MySociety::Config.get('TNDS_DIR', '')
+ # Iterate through the routes to be loaded, produce file of operators that can't
+ # be matched by operator code
+ Rake::Task['tnds:preload:list_unmatched_operators'].execute
+ Rake::Task['tnds:preload:load_unmatched_operators'].execute
+ Rake::Task['tnds:load:routes'].execute
+ end
desc 'Display a list of updates that have been made to instances of a model.
Default behaviour is to only show updates that have been marked as replayable.
- Specify ALL=1 to see all updates. Specify model class as MODEL=ModelName'
+ Specify ALL=1 to see all updates. Specify model class as MODEL=ModelName.
+ Specify a particular day as DATE=2012-04-23. Verbose flag set by VERBOSE=1'
task :show_updates => :environment do
check_for_model()
+ verbose = check_verbose()
model = ENV['MODEL'].constantize
only_replayable = (ENV['ALL'] == "1") ? false : true
- update_hash = get_updates(model, only_replayable=only_replayable, ENV['DATE'])
+ update_hash = get_updates(model, only_replayable=only_replayable, ENV['DATE'], verbose)
update_hash.each do |identity, changes|
identity_type = identity[:identity_type]
identity_hash = identity[:identity_hash]
@@ -137,6 +159,63 @@ namespace :update do
end
end
+ desc 'Generates an update file suitable for sending back to the source data provider from
+ the changes that have been made locally to a particular model. Verbose flag set by VERBOSE=1.'
+ task :create_update_file => :environment do
+ check_for_model()
+ verbose = check_verbose()
+ model = ENV['MODEL'].constantize
+ change_list = replay_updates(model, dryrun=true, verbose=verbose)
+ outfile = File.open("data/#{model}_changes_#{Date.today.to_s(:db)}.tsv", 'w')
+ headers = ['Change type']
+ identity_fields = model.data_generation_options_hash[:identity_fields]
+ significant_fields = model.data_generation_options_hash[:new_record_fields] +
+ model.data_generation_options_hash[:update_fields]
+ identity_fields.each do |identity_field|
+ headers << identity_field
+ end
+ headers += ["Attribute", "Old value", "New value", "Data"]
+ outfile.write(headers.join("\t")+"\n")
+ change_list.each do |change_info|
+ change_event = change_info[:event]
+ instance = change_info[:model]
+ if change_event == :update
+ attribute = change_info[:attribute]
+ from_value = change_info[:from_value]
+ to_value = change_info[:to_value]
+ if attribute == :generation_high
+ data_row = ["New"]
+ identity_fields.each do |identity_field|
+ data_row << ''
+ end
+ data_row += ['', '', '']
+ new_instance_info = {}
+ significant_fields.each do |field|
+ value = instance.send(field)
+ if value
+ new_instance_info[field] = value
+ end
+ end
+ data_row << new_instance_info.inspect
+ elsif attribute == :coords
+ elsif significant_fields.include?(attribute)
+ data_row = ["Update"]
+ identity_fields.each do |identity_field|
+ data_row << instance.send(identity_field)
+ end
+ data_row += [attribute, from_value, to_value]
+ end
+ elsif change_event == :destroy
+ data_row = ["Destroy"]
+ identity_fields.each do |identity_field|
+ data_row << instance.send(identity_field)
+ end
+ end
+ outfile.write(data_row.join("\t")+"\n") if data_row
+ end
+ outfile.close
+ end
+
desc 'Apply the replayable local updates for a model class that is versioned in data generations.
Runs in dryrun mode unless DRYRUN=0 is specified. Verbose flag set by VERBOSE=1'
task :replay_updates => :environment do
@@ -147,6 +226,18 @@ namespace :update do
replay_updates(model, dryrun, verbose)
end
+ desc "Mark as unreplayable local updates marked as replayable for a model class that refer to an
+ instance that does not exist and is not referred to by subsequent versions or don't contain
+ changes to any significant fields. Runs in dryrun mode unless DRYRUN=0 is specified. Verbose
+ flag set by VERBOSE=1"
+ task :mark_unreplayable => :environment do
+ check_for_model()
+ dryrun = check_dryrun()
+ verbose = check_verbose()
+ model = ENV['MODEL'].constantize
+ mark_unreplayable(model, dryrun, verbose)
+ end
+
desc 'Reorder any slugs that existed in the previous generation, but have been given a different
sequence by the arbitrary load order'
task :normalize_slug_sequences => :environment do
View
28 spec/models/route_spec.rb
@@ -27,8 +27,20 @@
:region_id => 1,
:status => 'ACT'
}
+ @default_attrs = { :name => 'A test route',
+ :status => 'ACT',
+ :number => 'ZZ9',
+ :transport_mode_id => 1 }
+ @model_type = Route
+ @expected_identity_hash = {}
+ @expected_temporary_identity_hash = { :id => nil }
end
+ it_should_behave_like "a model that exists in data generations"
+
+ it_should_behave_like "a model that exists in data generations and has slugs"
+
+
it "should create a new instance given valid attributes" do
route = Route.new(@valid_attributes)
route.valid?.should be_true
@@ -107,8 +119,8 @@
fixtures default_fixtures
- describe 'when looking for bus routes' do
-
+ describe 'when looking for bus routes' do
+
before do
@route = Route.new(:number => '807',
:transport_mode => transport_modes(:bus))
@@ -118,26 +130,26 @@
:to_stop => @new_stop,
:from_terminus => true)
end
-
+
it 'should include routes with the same number and one stop in common with the new route, with the same operator in the same admin area' do
@route.route_source_admin_areas.build({:operator_code => 'BUS',
:source_admin_area => admin_areas(:london)})
Route.find_existing_routes(@route).should include(routes(:number_807_bus))
end
-
+
it 'should include routes with the same number and one stop in common with the new route, with the same route operator' do
@route.route_operators.build({:operator => operators(:a_bus_company)})
Route.find_existing_routes(@route).should include(routes(:number_807_bus))
end
-
- it "should include routes with the same number and one stop in common with the new route, with one of the new route's operators" do
+
+ it "should include routes with the same number and one stop in common with the new route, with one of the new route's operators" do
@route.route_operators.build({:operator => operators(:a_bus_company)})
@route.route_operators.build({:operator => operators(:another_bus_company)})
Route.find_existing_routes(@route).should include(routes(:number_807_bus))
end
-
+
it 'should include routes with the same number, one stop area in common with the new route and the same operator code with no admin area' do
route_source_admin_area = routes(:number_807_bus).route_source_admin_areas.first
route_source_admin_area.source_admin_area_id = nil
@@ -176,7 +188,7 @@
end
end
-
+
describe 'when finding existing train routes' do
fixtures default_fixtures

No commit comments for this range

Something went wrong with that request. Please try again.