Merge pull request #994 from transitland/utilize-shape_dist_traveled

Utilize shape dist traveled

Gemfile

@@ -34,7 +34,7 @@ gem 'whenever', require: false # to manage crontab
 # data model
 gem 'squeel'
 gem 'enumerize'
-gem 'gtfs', github: 'transitland/gtfs', tag: 'f2d55ba1e84e557f91ae4376cf63a8ac955cd203'
+gem 'gtfs', github: 'transitland/gtfs', tag: 'c83e414378ef7fab246334b2dd1e60383986c1ba'
 gem 'rgeo-geojson'
 gem 'c_geohash', require: 'geohash'
 gem 'json-schema', '2.5.2' # running into problems with 2.6.0

Gemfile.lock

@@ -11,8 +11,8 @@ GIT
 
 GIT
   remote: git://github.com/transitland/gtfs.git
-  revision: f2d55ba1e84e557f91ae4376cf63a8ac955cd203
-  tag: f2d55ba1e84e557f91ae4376cf63a8ac955cd203
+  revision: c83e414378ef7fab246334b2dd1e60383986c1ba
+  tag: c83e414378ef7fab246334b2dd1e60383986c1ba
   specs:
     gtfs (1.0.1)
       multi_json
@@ -453,4 +453,4 @@ DEPENDENCIES
   whenever
 
 BUNDLED WITH
-   1.13.6
+   1.14.3

app/models/json_schemas/route_stop_pattern.json

@@ -20,11 +20,8 @@
         "type": ["number", "null"]
       }
     },
-    "isModified": {
-      "type": "boolean"
-    },
-    "isGenerated": {
-      "type": "boolean"
+    "geometrySource": {
+      "type": "string"
     },
     "addImportedFromFeeds": {
       "type": "array",

app/models/route_stop_pattern.rb

@@ -8,7 +8,6 @@
 #  tags                               :hstore
 #  stop_pattern                       :string           default([]), is an Array
 #  version                            :integer
-#  is_generated                       :boolean          default(FALSE)
 #  trips                              :string           default([]), is an Array
 #  identifiers                        :string           default([]), is an Array
 #  created_at                         :datetime         not null
@@ -17,6 +16,7 @@
 #  route_id                           :integer
 #  stop_distances                     :float            default([]), is an Array
 #  edited_attributes                  :string           default([]), is an Array
+#  geometry_source                    :string
 #
 # Indexes
 #
@@ -49,6 +49,9 @@ class RouteStopPattern < BaseRouteStopPattern
     :geometry_has_at_least_two_coords,
     :correct_stop_distances_length
 
+  extend Enumerize
+  enumerize :geometry_source, in: [:trip_stop_points, :shapes_txt, :shapes_txt_with_dist_traveled, :user_edited]
+
   def has_at_least_two_stops
     if stop_pattern.length < 2
       errors.add(:stop_pattern, 'RouteStopPattern needs at least 2 stops')
@@ -173,6 +176,25 @@ def fallback_distances(stops=nil)
     self.stop_distances.map!{ |distance| distance.round(DISTANCE_PRECISION) }
   end
 
+  def gtfs_shape_dist_traveled(stop_times, tl_stops, shape_distances_traveled)
+    # assumes stop times and shapes BOTH have shape_dist_traveled, and they're in the same units
+    # assumes the line geometry is not generated, and shape_points equals the rsp geometry.
+    self.stop_distances = []
+    stop_times.each_with_index do |st, i|
+      stop_onestop_id = self.stop_pattern[i]
+      # Find segment along shape points where stop shape_dist_traveled is between the two shape points' shape_dist_traveled
+      dist1, dist2 = shape_distances_traveled.zip(shape_distances_traveled[1..-1]).detect do |d1, d2|
+        st.shape_dist_traveled.to_f >= d1 && st.shape_dist_traveled.to_f <= d2
+      end
+      seg_index = shape_distances_traveled.index(dist1) # distances should always be increasing
+      cartesian_line = cartesian_cast(self[:geometry])
+      stop = tl_stops[i]
+      nearest_point_on_line = cartesian_line.closest_point_on_segment(cartesian_cast(stop[:geometry]), seg_index)
+      self.stop_distances << distance_along_line_to_nearest(cartesian_line, nearest_point_on_line, seg_index)
+    end
+    self.stop_distances.map!{ |distance| distance.round(DISTANCE_PRECISION) }
+  end
+
   def calculate_distances(stops=nil)
     if stops.nil?
       stop_hash = Hash[Stop.find_by_onestop_ids!(self.stop_pattern).map { |s| [s.onestop_id, s] }]
@@ -291,9 +313,10 @@ def self.create_from_gtfs(trip, route_onestop_id, stop_pattern, trip_stop_points
     )
     if shape_points.present?
       rsp.geometry = self.line_string(self.set_precision(shape_points))
+      rsp.geometry_source = shape_points.shape_dist_traveled.all? ? :shapes_txt_with_dist_traveled : :shapes_txt
     else
       rsp.geometry = self.line_string(self.set_precision(trip_stop_points))
-      rsp.is_generated = true
+      rsp.geometry_source = :trip_stop_points
     end
     onestop_id = OnestopId.handler_by_model(RouteStopPattern).new(
      route_onestop_id: route_onestop_id,

app/serializers/route_stop_pattern_serializer.rb

@@ -8,7 +8,6 @@
 #  tags                               :hstore
 #  stop_pattern                       :string           default([]), is an Array
 #  version                            :integer
-#  is_generated                       :boolean          default(FALSE)
 #  trips                              :string           default([]), is an Array
 #  identifiers                        :string           default([]), is an Array
 #  created_at                         :datetime         not null
@@ -17,6 +16,7 @@
 #  route_id                           :integer
 #  stop_distances                     :float            default([]), is an Array
 #  edited_attributes                  :string           default([]), is an Array
+#  geometry_source                    :string
 #
 # Indexes
 #
@@ -34,12 +34,13 @@ class RouteStopPatternSerializer < CurrentEntitySerializer
              :stop_pattern,
              :stop_distances,
              :geometry,
+             :geometry_source,
              :color,
-             :is_generated,
              :created_at,
              :updated_at,
              :trips,
              :tags
+
    def route_onestop_id
      object.route.onestop_id
    end

app/services/gtfs_graph.rb

@@ -183,13 +183,19 @@ def create_change_osr
   end
 
   def calculate_rsp_distances(rsps)
+    # TODO: move this into UpdateComputedAttributes Service
     graph_log "Calculating distances"
     rsps.each do |rsp|
       stops = rsp.stop_pattern.map { |onestop_id| find_by_onestop_id(onestop_id) }
       begin
-        if rsp.is_generated
+        # edited rsps will probably have a shape
+        if (rsp.geometry_source.eql?(:shapes_txt_with_dist_traveled))
+          # do nothing
+        elsif (rsp.geometry_source.eql?(:trip_stop_points) && rsp.edited_attributes.empty?)
           rsp.fallback_distances(stops=stops)
-        else
+        elsif (rsp.stop_distances.compact.empty? || rsp.issues.map(&:issue_type).include?(:distance_calculation_inaccurate))
+          # avoid writing over stop distances computed with shape_dist_traveled, or already computed somehow -
+          # unless if rsps have inaccurate stop distances, we'll allow a recomputation if there's a fix in place.
           rsp.calculate_distances(stops=stops)
         end
       rescue StandardError
@@ -533,9 +539,8 @@ def load_tl_route_stop_patterns
     @gtfs.trip_stop_times(trips=nil, filter_empty=true) do |trip,stop_times|
       tl_stops = stop_times.map { |stop_time| find_by_gtfs_entity(@gtfs.stop(stop_time.stop_id)) }
       stop_pattern = tl_stops.map(&:onestop_id)
-      stop_times_with_shape_dist_traveled += stop_times.count { |st| !st.shape_dist_traveled.to_s.empty? }
-      stop_times_count += stop_times.length
-      feed_shape_points = @gtfs.shape_line(trip.shape_id) || []
+      shape_line = @gtfs.shape_line(trip.shape_id)
+      feed_shape_points = shape_line || []
       tl_route = find_by_gtfs_entity(@gtfs.parents(trip).first)
       next if tl_route.nil?
       trip_stop_points = tl_stops.map { |s| s.geometry[:coordinates] }
@@ -546,14 +551,20 @@ def load_tl_route_stop_patterns
       if test_rsp.equal?(rsp)
         graph_log "   #{rsp.onestop_id}"
       end
+      shape_distances_traveled = shape_line.try(:shape_dist_traveled)
+      # assume stop_times' and shapes' shape_dist_traveled are in the same units (a condition required by GTFS). TODO: validate that.
+      if shape_distances_traveled
+        if stop_times.all?{ |st| st.shape_dist_traveled.present? } && shape_distances_traveled.all?(&:present?)
+          rsp.gtfs_shape_dist_traveled(stop_times, tl_stops, shape_distances_traveled)
+        end
+      end
       rsp.traversed_by = tl_route.onestop_id
       rsp.route = tl_route
       add_identifier(rsp, nil, trip.shape_id)
       @gtfs_to_onestop_id[trip] = rsp.onestop_id
       rsp.trips << trip.trip_id unless rsp.trips.include?(trip.trip_id)
       rsps << rsp
     end
-    graph_log "#{stop_times_with_shape_dist_traveled} stop times with shape_dist_traveled found out of #{stop_times_count} total stop times for feed #{@feed.onestop_id}" if stop_times_with_shape_dist_traveled > 0
     rsps
   end
 

config/initializers/rgeo.rb

@@ -11,6 +11,10 @@ def closest_point(target)
         nearest_locator(target).interpolate_point(factory)
       end
 
+      def closest_point_on_segment(target, seg_index)
+        _segments[seg_index].locator(target).interpolate_point(factory)
+      end
+
       def line_subset(start_index, stop_index)
         factory.line_string([_segments[start_index].s] + _segments[start_index..stop_index].map {|s| s.e})
       end

db/migrate/20170303213823_add_geometry_source_to_route_stop_pattern.rb

@@ -0,0 +1,8 @@
+class AddGeometrySourceToRouteStopPattern < ActiveRecord::Migration
+  def change
+    [:current, :old].each do |version|
+      add_column "#{version}_route_stop_patterns", :geometry_source, :string, index: true
+      remove_column "#{version}_route_stop_patterns", :is_generated
+    end
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170207173441) do
+ActiveRecord::Schema.define(version: 20170303213823) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -127,17 +127,17 @@
     t.string    "onestop_id"
     t.geography "geometry",                           limit: {:srid=>4326, :type=>"geometry", :geographic=>true}
     t.hstore    "tags"
-    t.string    "stop_pattern",                                                                                   default: [],                 array: true
+    t.string    "stop_pattern",                                                                                   default: [],              array: true
     t.integer   "version"
-    t.boolean   "is_generated",                                                                                   default: false
-    t.string    "trips",                                                                                          default: [],                 array: true
-    t.string    "identifiers",                                                                                    default: [],                 array: true
-    t.datetime  "created_at",                                                                                                     null: false
-    t.datetime  "updated_at",                                                                                                     null: false
+    t.string    "trips",                                                                                          default: [],              array: true
+    t.string    "identifiers",                                                                                    default: [],              array: true
+    t.datetime  "created_at",                                                                                                  null: false
+    t.datetime  "updated_at",                                                                                                  null: false
     t.integer   "created_or_updated_in_changeset_id"
     t.integer   "route_id"
-    t.float     "stop_distances",                                                                                 default: [],                 array: true
-    t.string    "edited_attributes",                                                                              default: [],                 array: true
+    t.float     "stop_distances",                                                                                 default: [],              array: true
+    t.string    "edited_attributes",                                                                              default: [],              array: true
+    t.string    "geometry_source"
   end
 
   add_index "current_route_stop_patterns", ["created_or_updated_in_changeset_id"], name: "c_rsp_cu_in_changeset", using: :btree
@@ -483,21 +483,21 @@
     t.string    "onestop_id"
     t.geography "geometry",                           limit: {:srid=>4326, :type=>"geometry", :geographic=>true}
     t.hstore    "tags"
-    t.string    "stop_pattern",                                                                                   default: [],                 array: true
+    t.string    "stop_pattern",                                                                                   default: [],              array: true
     t.integer   "version"
-    t.boolean   "is_generated",                                                                                   default: false
-    t.string    "trips",                                                                                          default: [],                 array: true
-    t.string    "identifiers",                                                                                    default: [],                 array: true
-    t.datetime  "created_at",                                                                                                     null: false
-    t.datetime  "updated_at",                                                                                                     null: false
+    t.string    "trips",                                                                                          default: [],              array: true
+    t.string    "identifiers",                                                                                    default: [],              array: true
+    t.datetime  "created_at",                                                                                                  null: false
+    t.datetime  "updated_at",                                                                                                  null: false
     t.integer   "created_or_updated_in_changeset_id"
     t.integer   "destroyed_in_changeset_id"
     t.integer   "route_id"
     t.string    "route_type"
     t.integer   "current_id"
-    t.float     "stop_distances",                                                                                 default: [],                 array: true
-    t.string    "edited_attributes",                                                                              default: [],                 array: true
+    t.float     "stop_distances",                                                                                 default: [],              array: true
+    t.string    "edited_attributes",                                                                              default: [],              array: true
     t.string    "action"
+    t.string    "geometry_source"
   end
 
   add_index "old_route_stop_patterns", ["created_or_updated_in_changeset_id"], name: "o_rsp_cu_in_changeset", using: :btree

spec/factories/feed_factory.rb

@@ -217,6 +217,25 @@
     end
   end
 
+  factory :feed_nj_path, parent: :feed, class: Feed do
+    onestop_id 'f-dr5r-panynjpath'
+    url 'http://data.trilliumtransit.com/gtfs/path-nj-us/path-nj-us.zip'
+    version 1
+    after :create do |feed, evaluator|
+      operator = create(
+        :operator,
+        name: 'Port Authority Trans-Hudson',
+        onestop_id: 'o-dr5r-path',
+        timezone: 'America/New_York',
+        website: 'http://www.panynj.gov/',
+      )
+      feed.operators_in_feed.create(
+        operator: operator,
+        gtfs_agency_id: '151'
+      )
+    end
+  end
+
   factory :feed_example, parent: :feed, class: Feed do
     onestop_id 'f-9qs-example'
     url 'http://www.bart.gov/dev/schedules/google_transit.zip'

spec/factories/feed_version_factory.rb

@@ -96,6 +96,11 @@
       association :feed, factory: :feed_rome
     end
 
+    factory :feed_version_nj_path do
+      file { File.open(Rails.root.join('spec/support/example_gtfs_archives/path-nj-us.zip')) }
+      association :feed, factory: :feed_nj_path
+    end
+
     factory :feed_version_example do
       file { File.open(Rails.root.join('spec/support/example_gtfs_archives/example.zip')) }
       association :feed, factory: :feed_example

spec/factories/route_stop_pattern_factory.rb

@@ -8,7 +8,6 @@
 #  tags                               :hstore
 #  stop_pattern                       :string           default([]), is an Array
 #  version                            :integer
-#  is_generated                       :boolean          default(FALSE)
 #  trips                              :string           default([]), is an Array
 #  identifiers                        :string           default([]), is an Array
 #  created_at                         :datetime         not null
@@ -17,6 +16,7 @@
 #  route_id                           :integer
 #  stop_distances                     :float            default([]), is an Array
 #  edited_attributes                  :string           default([]), is an Array
+#  geometry_source                    :string
 #
 # Indexes
 #