Merging stop_order into develop branch

Examples/RLvsTBHPexample2/input/pathweight_ft.txt

@@ -5,5 +5,5 @@ all        access           walk           walk_access  preferred_delay_min 2.0
 all        egress           walk           walk_egress  time_min            0.0
 all        egress           walk           walk_egress  preferred_delay_min 2.0
 all        transit          local_bus      local_bus    in_vehicle_time_min 1.0
-all        transit          local_bus      local_bus    wait_time_min       2.0
+all        transit          local_bus      local_bus    wait_time_min       2.01
 all        transit          local_bus      local_bus    transfer_penalty    0.0

Examples/test_network/input/config_ft.txt

@@ -7,6 +7,7 @@ output_passenger_trajectories = True
 time_window                   = 30
 create_skims                  = False
 stochastic_dispersion         = 0.2
+stochastic_max_stop_process_count = 1
 stochastic_pathset_size       = 1000
 capacity_constraint           = False
 trace_person_ids              = ['bunnies']

fasttrips/Assignment.py

@@ -17,14 +17,15 @@
 import numpy,pandas
 import _fasttrips
 
-from .Logger    import FastTripsLogger, setupLogging
-from .Passenger import Passenger
-from .Path      import Path
-from .Stop      import Stop
-from .TAZ       import TAZ
-from .Transfer  import Transfer
-from .Trip      import Trip
-from .Util      import Util
+from .Logger      import FastTripsLogger, setupLogging
+from .Passenger   import Passenger
+from .Path        import Path
+from .Performance import Performance
+from .Stop        import Stop
+from .TAZ         import TAZ
+from .Transfer    import Transfer
+from .Trip        import Trip
+from .Util        import Util
 
 class Assignment:
     """
@@ -86,6 +87,12 @@ class Assignment:
     #: If unknown use a value between 0.5 and 1. Float.
     STOCH_DISPERSION                = None
 
+    #: Route choice configuration: How many times max times should we process a stop
+    #: during labeling?  Use -1 to specify no max.  Int.
+    #: Setting this to a positive value may increase runtime but may decrease
+    #: pathset quality. (Todo: test/quantify this.)
+    STOCH_MAX_STOP_PROCESS_COUNT    = None
+
     #: Route choice configuration: How many stochastic paths will we generate
     #: (not necessarily unique) to define a path choice set?  Int.
     STOCH_PATHSET_SIZE              = None
@@ -177,6 +184,7 @@ def read_configuration(input_network_dir, input_demand_dir):
                       'skim_start_time'                 :'5:00',
                       'skim_end_time'                   :'10:00',
                       'stochastic_dispersion'           :1.0,
+                      'stochastic_max_stop_process_count':-1,
                       'stochastic_pathset_size'         :1000,
                       'capacity_constraint'             :False,
                       'trace_person_ids'                :'None',
@@ -207,6 +215,7 @@ def read_configuration(input_network_dir, input_demand_dir):
                                                    parser.get       ('fasttrips','skim_end_time'),'%H:%M')
         Assignment.STOCH_DISPERSION              = parser.getfloat  ('fasttrips','stochastic_dispersion')
         Assignment.STOCH_PATHSET_SIZE            = parser.getint    ('fasttrips','stochastic_pathset_size')
+        Assignment.STOCH_MAX_STOP_PROCESS_COUNT  = parser.getint    ('fasttrips','stochastic_max_stop_process_count')
         Assignment.CAPACITY_CONSTRAINT           = parser.getboolean('fasttrips','capacity_constraint')
         Assignment.TRACE_PERSON_IDS         = eval(parser.get       ('fasttrips','trace_person_ids'))
         Assignment.PREPEND_ROUTE_ID_TO_TRIP_ID   = parser.getboolean('fasttrips','prepend_route_id_to_trip_id')
@@ -246,6 +255,7 @@ def write_configuration(output_dir):
         parser.set('fasttrips','skim_start_time',               Assignment.SKIM_START_TIME.strftime('%H:%M'))
         parser.set('fasttrips','skim_end_time',                 Assignment.SKIM_END_TIME.strftime('%H:%M'))
         parser.set('fasttrips','stochastic_dispersion',         '%f' % Assignment.STOCH_DISPERSION)
+        parser.set('fasttrips','stochastic_max_stop_process_count', '%d' % Assignment.STOCH_MAX_STOP_PROCESS_COUNT)
         parser.set('fasttrips','stochastic_pathset_size',       '%d' % Assignment.STOCH_PATHSET_SIZE)
         parser.set('fasttrips','capacity_constraint',           'True' if Assignment.CAPACITY_CONSTRAINT else 'False')
         parser.set('fasttrips','trace_person_ids',              '%s' % str(Assignment.TRACE_PERSON_IDS))
@@ -279,7 +289,8 @@ def initialize_fasttrips_extension(process_number, output_dir, FT):
         _fasttrips.initialize_parameters(Assignment.TIME_WINDOW.total_seconds()/60.0,
                                          Assignment.BUMP_BUFFER.total_seconds()/60.0,
                                          Assignment.STOCH_PATHSET_SIZE,
-                                         Assignment.STOCH_DISPERSION)
+                                         Assignment.STOCH_DISPERSION,
+                                         Assignment.STOCH_MAX_STOP_PROCESS_COUNT)
 
     @staticmethod
     def set_fasttrips_bump_wait(bump_wait_df):
@@ -417,11 +428,13 @@ def generate_paths(FT, output_dir, iteration):
                         trace_person = True
 
                     # do the work
-                    (cost, return_states) = Assignment.find_trip_based_path(iteration, FT, trip_path,
-                                                                            Assignment.ASSIGNMENT_TYPE==Assignment.ASSIGNMENT_TYPE_STO_ASGN,
-                                                                            trace=trace_person)
+                    (cost, return_states, perf_dict) = \
+                        Assignment.find_trip_based_path(iteration, FT, trip_path,
+                                                        Assignment.ASSIGNMENT_TYPE==Assignment.ASSIGNMENT_TYPE_STO_ASGN,
+                                                        trace=trace_person)
                     trip_path.states = return_states
                     trip_path.cost   = cost
+                    FT.performance.add_info(iteration, trip_list_id, perf_dict)
 
                     if trip_path.path_found():
                         num_paths_found_now += 1
@@ -454,6 +467,10 @@ def generate_paths(FT, output_dir, iteration):
                         path            = FT.passengers.get_path(trip_list_id)
                         path.cost       = result[1]
                         path.states     = result[2]
+                        perf_dict       = result[3]
+
+                        FT.performance.add_info(iteration, trip_list_id, perf_dict)
+
                         if path.path_found():
                             num_paths_found_now += 1
 
@@ -492,6 +509,7 @@ def generate_paths(FT, output_dir, iteration):
                                  int( time_elapsed.total_seconds() / 3600),
                                  int( (time_elapsed.total_seconds() % 3600) / 60),
                                  time_elapsed.total_seconds() % 60))
+
         return num_paths_found_now + num_paths_found_prev
 
 
@@ -503,7 +521,15 @@ def find_trip_based_path(iteration, FT, path, hyperpath, trace):
         Will do so either backwards (destination to origin) if :py:attr:`Path.direction` is :py:attr:`Path.DIR_OUTBOUND`
         or forwards (origin to destination) if :py:attr:`Path.direction` is :py:attr:`Path.DIR_INBOUND`.
 
-        Returns (path cost, return_states).
+        Returns (path cost,
+                 return_states,
+                 performance_dict)
+
+        Where performance_dict includes:
+                 number of label iterations,
+                 max number of times a stop was processed,
+                 seconds spent in labeling,
+                 seconds spend in enumeration
 
         :param iteration: The pathfinding iteration we're on
         :type  iteration: int
@@ -520,7 +546,9 @@ def find_trip_based_path(iteration, FT, path, hyperpath, trace):
         """
         # FastTripsLogger.debug("C++ extension start")
         # send it to the C++ extension
-        (ret_ints, ret_doubles, path_cost) = \
+        (ret_ints, ret_doubles, path_cost,
+         label_iterations, max_label_process_count,
+         seconds_labeling, seconds_enumerating) = \
             _fasttrips.find_path(iteration, path.person_id_num, path.trip_list_id_num, hyperpath,
                                  path.user_class, path.access_mode, path.transit_mode, path.egress_mode,
                                  path.o_taz_num, path.d_taz_num,
@@ -571,7 +599,14 @@ def find_trip_based_path(iteration, FT, path, hyperpath, trace):
                               datetime.timedelta(minutes=ret_doubles[index,3]),             # cost
                               midnight + datetime.timedelta(minutes=ret_doubles[index,4])   # arrival/departure time
                               ] ) )
-        return (path_cost, return_states)
+        perf_dict = { \
+            Performance.PERFORMANCE_COLUMN_LABEL_ITERATIONS      : label_iterations,
+            Performance.PERFORMANCE_COLUMN_MAX_STOP_PROCESS_COUNT: max_label_process_count,
+            Performance.PERFORMANCE_COLUMN_TIME_LABELING_MS      : seconds_labeling,
+            Performance.PERFORMANCE_COLUMN_TIME_ENUMERATING_MS   : seconds_enumerating,
+            Performance.PERFORMANCE_COLUMN_TRACED                : trace,
+        }
+        return (path_cost, return_states, perf_dict)
 
     @staticmethod
     def read_assignment_results(output_dir, iteration):
@@ -1261,8 +1296,8 @@ def find_trip_based_paths_process_worker(iteration, worker_num, input_network_di
             trace_person = True
 
         try:
-            (cost, return_states) = Assignment.find_trip_based_path(iteration, worker_FT, path, hyperpath, trace=trace_person)
-            done_queue.put( (path.trip_list_id_num, cost, return_states) )
+            (cost, return_states, perf_dict) = Assignment.find_trip_based_path(iteration, worker_FT, path, hyperpath, trace=trace_person)
+            done_queue.put( (path.trip_list_id_num, cost, return_states, perf_dict) )
         except:
             FastTripsLogger.exception('Exception')
             # call it a day

fasttrips/FastTrips.py

@@ -17,14 +17,15 @@
 import pandas
 import transitfeed
 
-from .Assignment import Assignment
-from .Logger import FastTripsLogger, setupLogging
-from .Passenger import Passenger
-from .Route import Route
-from .Stop import Stop
-from .TAZ import TAZ
-from .Transfer import Transfer
-from .Trip import Trip
+from .Assignment  import Assignment
+from .Logger      import FastTripsLogger, setupLogging
+from .Passenger   import Passenger
+from .Performance import Performance
+from .Route       import Route
+from .Stop        import Stop
+from .TAZ         import TAZ
+from .Transfer    import Transfer
+from .Trip        import Trip
 
 class FastTrips:
     """
@@ -190,16 +191,20 @@ def combine_pathset_files(self):
 
         if pathset_init:
             # sort it by iteration, trip_id_num
-            pathsets_df.sort(columns=['iteration','trip_list_id_num'], inplace=True)
+            pathsets_df.sort_values(by=['iteration','trip_list_id_num'], inplace=True)
             # write it
             pathset_filename = os.path.join(self.output_dir, FastTrips.PATHSET_LOG % "")
             pathsets_df.to_csv(pathset_filename, sep=" ", index=False)
             FastTripsLogger.info("Wrote %d lines to %s" % (len(pathsets_df), pathset_filename))
 
 
     def run_assignment(self, output_dir):
+        # Initialize performance results
+        self.performance = Performance()
+
         # Do it!
         Assignment.assign_paths(output_dir, self)
 
         self.combine_pathset_files()
+        self.performance.write(output_dir)
 

fasttrips/Path.py

@@ -327,7 +327,7 @@ def write_paths(passengers_df, output_dir):
                                             walktimes_str], axis=1)
 
         print_passengers_df.reset_index(inplace=True)
-        print_passengers_df.sort(columns=['trip_list_id_num'], inplace=True)
+        print_passengers_df.sort_values(by=['trip_list_id_num'], inplace=True)
 
         print_passengers_df.rename(columns=
            {'pathmode'          :'mode',
@@ -359,7 +359,7 @@ def write_path_times(pax_exp_df, output_dir):
         """
         # reset columns
         print_pax_exp_df = pax_exp_df.reset_index()
-        print_pax_exp_df.sort(columns=['trip_list_id_num'], inplace=True)
+        print_pax_exp_df.sort_values(by=['trip_list_id_num'], inplace=True)
 
         print_pax_exp_df['A_time_str'] = print_pax_exp_df['A_time'].apply(Util.datetime64_formatter)
         print_pax_exp_df['B_time_str'] = print_pax_exp_df['B_time'].apply(Util.datetime64_formatter)

fasttrips/Performance.py

@@ -0,0 +1,80 @@
+__copyright__ = "Copyright 2016 Contributing Entities"
+__license__   = """
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+"""
+import datetime, os
+import pandas
+
+from .Logger    import FastTripsLogger
+from .Passenger import Passenger
+
+class Performance:
+    """
+    Performance class.  Keeps track of performance information
+    (time spent, number of labeling iterations, etc) related to pathfinding
+    in Fast-Trips.
+    """
+    #: Performance column: Iteration
+    PERFORMANCE_COLUMN_ITERATION              = "iteration"
+    #: Performance column: Trip list ID num
+    PERFORMANCE_COLUMN_TRIP_LIST_ID_NUM       = Passenger.TRIP_LIST_COLUMN_TRIP_LIST_ID_NUM
+    #: Performance column: Number of label iterations
+    PERFORMANCE_COLUMN_LABEL_ITERATIONS       = "label iterations"
+    #: Performance column: Maximum number of times a stop was processed
+    PERFORMANCE_COLUMN_MAX_STOP_PROCESS_COUNT = "max stop process count"
+    #: Performance column: Time spent labeling (timedelta)
+    PERFORMANCE_COLUMN_TIME_LABELING          = "time labeling"
+    #: Performance column: Time spent labeling (milliseconds)
+    PERFORMANCE_COLUMN_TIME_LABELING_MS       = "time labeling milliseconds"
+    #: Performance column: Time spent enumerating (timedelta)
+    PERFORMANCE_COLUMN_TIME_ENUMERATING       = "time enumerating"
+    #: Performance column: Time spent enumerating (milliseconds)
+    PERFORMANCE_COLUMN_TIME_ENUMERATING_MS    = "time enumerating milliseconds"
+    #: Performance column: Traced, since this affects performance
+    PERFORMANCE_COLUMN_TRACED                 = "traced"
+
+    #: File with to write performance results
+    OUTPUT_PERFORMANCE_FILE                   = 'ft_output_performance.txt'
+
+    def __init__(self):
+        """
+        Constructor.  Initialize empty dataframe for performance info.
+        """
+        self.performance_df = pandas.DataFrame(columns=[Performance.PERFORMANCE_COLUMN_ITERATION,
+                                                        Performance.PERFORMANCE_COLUMN_TRIP_LIST_ID_NUM,
+                                                        Performance.PERFORMANCE_COLUMN_LABEL_ITERATIONS,
+                                                        Performance.PERFORMANCE_COLUMN_MAX_STOP_PROCESS_COUNT,
+                                                        Performance.PERFORMANCE_COLUMN_TIME_LABELING,
+                                                        Performance.PERFORMANCE_COLUMN_TIME_ENUMERATING])
+
+    def add_info(self, iteration, trip_list_id_num, perf_dict):
+        """
+        Add this row to the performance dataframe.
+        Assumes time values are in milliseconds.
+        """
+        perf_dict[Performance.PERFORMANCE_COLUMN_ITERATION       ] = iteration
+        perf_dict[Performance.PERFORMANCE_COLUMN_TRIP_LIST_ID_NUM] = trip_list_id_num
+
+        # convert milliseconds time to timedeltas
+        perf_dict[Performance.PERFORMANCE_COLUMN_TIME_LABELING   ] = datetime.timedelta(milliseconds=perf_dict[Performance.PERFORMANCE_COLUMN_TIME_LABELING_MS   ])
+        perf_dict[Performance.PERFORMANCE_COLUMN_TIME_ENUMERATING] = datetime.timedelta(milliseconds=perf_dict[Performance.PERFORMANCE_COLUMN_TIME_ENUMERATING_MS])
+
+        self.performance_df = self.performance_df.append(perf_dict, ignore_index=True)
+
+    def write(self, output_dir):
+        """
+        Writes the results to OUTPUT_PERFORMANCE_FILE to a tab-delimited file.
+        """
+        output_filename = os.path.join(output_dir, Performance.OUTPUT_PERFORMANCE_FILE)
+        self.performance_df.to_csv(output_filename, sep="\t", index=False, date_format="")
+        FastTripsLogger.info("Wrote performance info to %s" % output_filename)

fasttrips/Trip.py

@@ -495,7 +495,7 @@ def calculate_headways(trips_df):
                                    Trip.STOPTIMES_COLUMN_STOP_SEQUENCE]].groupby([Trip.STOPTIMES_COLUMN_STOP_ID,
                                                                                  Trip.TRIPS_COLUMN_ROUTE_ID])
 
-        stop_group_df = stop_group.apply(lambda x: x.sort(Trip.STOPTIMES_COLUMN_DEPARTURE_TIME))
+        stop_group_df = stop_group.apply(lambda x: x.sort_values(Trip.STOPTIMES_COLUMN_DEPARTURE_TIME))
         # set headway, in minutes
         stop_group_shift_df = stop_group_df.shift()
         stop_group_df['headway'] = (stop_group_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME] - stop_group_shift_df[Trip.STOPTIMES_COLUMN_DEPARTURE_TIME])/numpy.timedelta64(1,'m')

fasttrips/__init__.py

@@ -18,11 +18,13 @@
 from .Logger import FastTripsLogger, setupLogging
 from .Passenger import Passenger
 from .Path import Path
+from .Performance import Performance
 from .Route import Route
 from .Stop import Stop
 from .TAZ import TAZ
 from .Transfer import Transfer
 from .Trip import Trip
+from .Util import Util
 
 __all__ = [
     'Event',