Merge pull request #2919 from entur/otp2_fast_transitlayer_updater

Faster TransitLayer realtime updater

docs/Changelog.md

@@ -17,6 +17,7 @@
 - Add command line parameter for building partial graphs (#2583)
 - Refactor GenericLocation/AStar/RoutingContext to allow multiple start vertices (#2887) 
 - New Transit search algorithm, Raptor, replaces the AStar for all transit searches. 
+- Update only the relevant parts of the TransitLayer each time an update is applied (#2918)
 - Ability to switch off the fare service(#2912).
 - Limit the transit service period(#2925).
 - Removed unwanted cost added for wait time between access and transit with RangeRaptor (#2927)

src/ext/java/org/opentripplanner/ext/examples/updater/ExampleGraphUpdater.java

@@ -76,4 +76,8 @@ public void teardown() {
         LOG.info("Teardown example updater");
     }
 
+    @Override
+    public String getName() {
+        return "ExampleGraphUpdater";
+    }
 }

src/ext/java/org/opentripplanner/ext/readiness_endpoint/ActuatorAPI.java

@@ -1,18 +1,23 @@
 package org.opentripplanner.ext.readiness_endpoint;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.opentripplanner.standalone.server.OTPServer;
 import org.opentripplanner.standalone.server.Router;
 
 import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.util.Collection;
 
 
 @Path("/actuators")
 @Produces(MediaType.APPLICATION_JSON) // One @Produces annotation for all endpoints.
 public class ActuatorAPI {
 
+    private static final Logger LOG = LoggerFactory.getLogger(ActuatorAPI.class);
+
     private final Router router;
 
     public ActuatorAPI(@Context OTPServer otpServer) {
@@ -44,6 +49,19 @@ public Response actuator() {
     @GET
     @Path("/health")
     public Response health() {
+        if (router.graph.updaterManager != null) {
+            Collection<String> waitingUpdaters = router.graph.updaterManager.waitingUpdaters();
+
+            if (!waitingUpdaters.isEmpty()) {
+                LOG.info("Graph ready, waiting for updaters: {}", waitingUpdaters);
+                throw new WebApplicationException(Response
+                    .status(Response.Status.NOT_FOUND)
+                    .entity("Graph ready, waiting for updaters: " + waitingUpdaters + "\n")
+                    .type("text/plain")
+                    .build());
+            }
+        }
+
         return Response.status(Response.Status.OK).entity(
             "{\n"
             + "  \"status\" : \"UP\""

src/ext/java/org/opentripplanner/ext/siri/SiriHttpUtils.java

@@ -12,11 +12,19 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
 public class SiriHttpUtils extends HttpUtils {
 
-    public static InputStream postData(String url, String xmlData, int timeout) throws IOException {
+    public static InputStream postData(
+        String url,
+        String xmlData,
+        int timeout,
+        Map<String, String> headers
+    ) throws IOException {
+
         HttpPost httppost = new HttpPost(url);
         if (xmlData != null) {
             httppost.setEntity(new StringEntity(xmlData, ContentType.APPLICATION_XML));
@@ -34,6 +42,17 @@ public static InputStream postData(String url, String xmlData, int timeout) thro
         return entity.getContent();
     }
 
+    /**
+     * Gets a unique ET-Client-Name HTTP header for this instance of OTP.
+     */
+    public static String getUniqueETClientName(String postFix) {
+        String hostname = System.getenv("HOSTNAME");
+        if (hostname == null) {
+            hostname = "otp-"+ UUID.randomUUID().toString();
+        }
+        return hostname + postFix;
+    }
+
     private static HttpClient getClient(int socketTimeout, int connectionTimeout) {
         HttpClient httpClient = HttpClientBuilder.create()
                 .setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(socketTimeout).build())

src/ext/java/org/opentripplanner/ext/siri/SiriTimetableSnapshotSource.java

@@ -13,6 +13,8 @@
 import org.opentripplanner.model.Trip;
 import org.opentripplanner.model.TripPattern;
 import org.opentripplanner.model.calendar.ServiceDate;
+import org.opentripplanner.routing.algorithm.raptor.transit.TransitLayer;
+import org.opentripplanner.routing.algorithm.raptor.transit.mappers.TransitLayerUpdater;
 import org.opentripplanner.routing.graph.Graph;
 import org.opentripplanner.routing.graph.GraphIndex;
 import org.opentripplanner.routing.trippattern.RealTimeState;
@@ -116,9 +118,15 @@ public class SiriTimetableSnapshotSource implements TimetableSnapshotProvider {
 
     public SiriFuzzyTripMatcher siriFuzzyTripMatcher;
 
+    private TransitLayer realtimeTransitLayer;
+
+    private TransitLayerUpdater transitLayerUpdater;
+
     public SiriTimetableSnapshotSource(final Graph graph) {
         timeZone = graph.getTimeZone();
         graphIndex = graph.index;
+        realtimeTransitLayer = graph.getRealtimeTransitLayer();
+        transitLayerUpdater = graph.transitLayerUpdater;
 
         // Create dummy agency for added trips
         Agency dummy = new Agency();
@@ -160,7 +168,7 @@ private TimetableSnapshot getTimetableSnapshot(final boolean force) {
         if (force || now - lastSnapshotTime > maxSnapshotFrequency) {
             if (force || buffer.isDirty()) {
                 LOG.debug("Committing {}", buffer.toString());
-                snapshot = buffer.commit(force);
+                snapshot = buffer.commit(transitLayerUpdater, force);
             } else {
                 LOG.debug("Buffer was unchanged, keeping old snapshot.");
             }
@@ -394,7 +402,7 @@ private boolean handleModifiedTrip(Graph graph, String feedId, VehicleActivitySt
         }
         boolean success = false;
         for (TripPattern pattern : patterns) {
-            if (handleTripPatternUpdate(graph, feedId, pattern, activity, trip, serviceDate)) {
+            if (handleTripPatternUpdate(graph, pattern, activity, trip, serviceDate)) {
                 success = true;
             }
         }
@@ -405,7 +413,7 @@ private boolean handleModifiedTrip(Graph graph, String feedId, VehicleActivitySt
         return success;
     }
 
-    private boolean handleTripPatternUpdate(Graph graph, String feedId,  TripPattern pattern, VehicleActivityStructure activity, Trip trip, ServiceDate serviceDate) {
+    private boolean handleTripPatternUpdate(Graph graph, TripPattern pattern, VehicleActivityStructure activity, Trip trip, ServiceDate serviceDate) {
 
         // Apply update on the *scheduled* time table and set the updated trip times in the buffer
         Timetable currentTimetable = getCurrentTimetable(pattern, serviceDate);
@@ -414,7 +422,7 @@ private boolean handleTripPatternUpdate(Graph graph, String feedId,  TripPattern
             return false;
         }
 
-        final boolean success = buffer.update(feedId, pattern, updatedTripTimes, serviceDate);
+        final boolean success = buffer.update(pattern, updatedTripTimes, serviceDate);
 
         return success;
     }
@@ -803,7 +811,7 @@ private boolean handleModifiedTrip(Graph graph, String feedId, EstimatedVehicleJ
                             result = result | addTripToGraphAndBuffer(feedId, graph, trip, modifiedStopTimes, modifiedStops, tripTimes, serviceDate);
                         }
                     } else {
-                        result = result | buffer.update(feedId, pattern, tripTimes, serviceDate);
+                        result = result | buffer.update(pattern, tripTimes, serviceDate);
                     }
 
                     LOG.debug("Applied realtime data for trip {}", trip.getId().getId());
@@ -879,12 +887,12 @@ private boolean addTripToGraphAndBuffer(final String feedId, final Graph graph,
         pattern.scheduledTimetable.tripTimes.clear();
 
         // Add to buffer as-is to include it in the 'lastAddedTripPattern'
-        buffer.update(feedId, pattern, updatedTripTimes, serviceDate);
+        buffer.update(pattern, updatedTripTimes, serviceDate);
 
         //TODO - SIRI: Add pattern to index?
 
         // Add new trip times to the buffer
-        final boolean success = buffer.update(feedId, pattern, updatedTripTimes, serviceDate);
+        final boolean success = buffer.update(pattern, updatedTripTimes, serviceDate);
         return success;
     }
 
@@ -909,7 +917,7 @@ private boolean cancelScheduledTrip(String feedId, String tripId, final ServiceD
             } else {
                 final TripTimes newTripTimes = new TripTimes(timetable.getTripTimes(tripIndex));
                 newTripTimes.cancel();
-                buffer.update(feedId, pattern, newTripTimes, serviceDate);
+                buffer.update(pattern, newTripTimes, serviceDate);
                 success = true;
             }
         }
@@ -929,7 +937,7 @@ private boolean cancelScheduledTrip(String feedId, String tripId, final ServiceD
     private boolean cancelPreviouslyAddedTrip(final String feedId, final String tripId, final ServiceDate serviceDate) {
         boolean success = false;
 
-        final TripPattern pattern = buffer.getLastAddedTripPattern(feedId, tripId, serviceDate);
+        final TripPattern pattern = buffer.getLastAddedTripPattern(new FeedScopedId(feedId, tripId), serviceDate);
         if (pattern != null) {
             // Cancel trip times for this trip in this pattern
             final Timetable timetable = buffer.resolve(pattern, serviceDate);
@@ -939,7 +947,7 @@ private boolean cancelPreviouslyAddedTrip(final String feedId, final String trip
             } else {
                 final TripTimes newTripTimes = new TripTimes(timetable.getTripTimes(tripIndex));
                 newTripTimes.cancel();
-                buffer.update(feedId, pattern, newTripTimes, serviceDate);
+                buffer.update(pattern, newTripTimes, serviceDate);
 //                buffer.removeLastAddedTripPattern(feedId, tripId, serviceDate);
                 success = true;
             }
@@ -1022,7 +1030,7 @@ private Set<TripPattern> getPatternsForTrip(Set<Trip> matches, VehicleActivitySt
 
                 if (firstStopIsMatch & lastStopIsMatch) {
                     // Origin and destination matches
-                    TripPattern lastAddedTripPattern = buffer.getLastAddedTripPattern(tripPattern.getFeedId(), currentTrip.getId().getId(), realTimeReportedServiceDate);
+                    TripPattern lastAddedTripPattern = buffer.getLastAddedTripPattern(currentTrip.getId(), realTimeReportedServiceDate);
                     if (lastAddedTripPattern != null) {
                         patterns.add(lastAddedTripPattern);
                     } else {
@@ -1078,7 +1086,7 @@ private TripPattern getPatternForTrip(Trip trip, EstimatedVehicleJourney journey
 
             TripPattern lastAddedTripPattern = null;
             if (getTimetableSnapshot() != null) {
-                lastAddedTripPattern  = getTimetableSnapshot().getLastAddedTripPattern(trip.getId().getFeedId(), trip.getId().getId(), journeyDate);
+                lastAddedTripPattern  = getTimetableSnapshot().getLastAddedTripPattern(trip.getId(), journeyDate);
             }
 
             TripPattern tripPattern;

src/ext/java/org/opentripplanner/ext/siri/SiriTripPatternCache.java

@@ -157,7 +157,7 @@ private String generateUniqueTripPatternCode(TripPattern tripPattern) {
             counter++;
         }
         // OBA library uses underscore as separator, we're moving toward colon.
-        String code = String.format("%s:%s:%s:rt#%d", routeId.getFeedId(), routeId.getId(), direction, counter);
+        String code = String.format("%s:%s:rt#%d", routeId.getId(), direction, counter);
         return code;
     }
 

src/ext/java/org/opentripplanner/ext/siri/updater/SiriETHttpTripUpdateSource.java

@@ -11,6 +11,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.time.ZonedDateTime;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.UUID;
 
 public class SiriETHttpTripUpdateSource implements EstimatedTimetableSource, JsonConfigurable {
@@ -38,6 +40,8 @@ public class SiriETHttpTripUpdateSource implements EstimatedTimetableSource, Jso
 
     private int previewIntervalMillis = -1;
 
+    private static Map<String, String> requestHeaders = new HashMap<>();
+
     @Override
     public void configure(Graph graph, JsonNode config) throws Exception {
         String url = config.path("url").asText();
@@ -61,6 +65,8 @@ public void configure(Graph graph, JsonNode config) throws Exception {
         if (previewIntervalMinutes > 0) {
             this.previewIntervalMillis = 1000*60*previewIntervalMinutes;
         }
+
+        requestHeaders.put("ET-Client-Name", SiriHttpUtils.getUniqueETClientName("-ET"));
     }
 
     @Override
@@ -75,7 +81,7 @@ public Siri getUpdates() {
             creating =  System.currentTimeMillis()-t1;
             t1 = System.currentTimeMillis();
 
-            InputStream is = SiriHttpUtils.postData(url, etServiceRequest, timeout);
+            InputStream is = SiriHttpUtils.postData(url, etServiceRequest, timeout, requestHeaders);
             if (is != null) {
                 // Decode message
                 fetching = System.currentTimeMillis()-t1;

src/ext/java/org/opentripplanner/ext/siri/updater/SiriSXUpdater.java

@@ -17,6 +17,8 @@
 
 import java.io.InputStream;
 import java.time.ZonedDateTime;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.UUID;
 
 public class SiriSXUpdater extends PollingGraphUpdater {
@@ -42,6 +44,8 @@ public class SiriSXUpdater extends PollingGraphUpdater {
 
     private int timeout;
 
+    private static Map<String, String> requestHeaders = new HashMap<>();
+
 
     @Override
     public void setGraphUpdaterManager(GraphUpdaterManager updaterManager) {
@@ -88,6 +92,8 @@ protected void configurePolling(Graph graph, JsonNode config) throws Exception {
 
         this.fuzzyTripMatcher = new SiriFuzzyTripMatcher(graph.index);
 
+        requestHeaders.put("ET-Client-Name", SiriHttpUtils.getUniqueETClientName("-SX"));
+
         LOG.info("Creating real-time alert updater (SIRI SX) running every {} seconds : {}", pollingPeriodSeconds, url);
     }
 
@@ -122,7 +128,7 @@ private Siri getUpdates() {
             creating = System.currentTimeMillis()-t1;
             t1 = System.currentTimeMillis();
 
-            InputStream is = SiriHttpUtils.postData(url, sxServiceRequest, timeout);
+            InputStream is = SiriHttpUtils.postData(url, sxServiceRequest, timeout, requestHeaders);
 
             fetching = System.currentTimeMillis()-t1;
             t1 = System.currentTimeMillis();

src/ext/java/org/opentripplanner/ext/siri/updater/SiriVMHttpTripUpdateSource.java

@@ -10,6 +10,8 @@
 
 import java.io.InputStream;
 import java.time.ZonedDateTime;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.UUID;
 
 public class SiriVMHttpTripUpdateSource implements VehicleMonitoringSource, JsonConfigurable {
@@ -34,6 +36,8 @@ public class SiriVMHttpTripUpdateSource implements VehicleMonitoringSource, Json
     private String requestorRef;
     private int timeout;
 
+    private static Map<String, String> requestHeaders = new HashMap<>();
+
     @Override
     public void configure(Graph graph, JsonNode config) throws Exception {
         String url = config.path("url").asText();
@@ -53,6 +57,7 @@ public void configure(Graph graph, JsonNode config) throws Exception {
             this.timeout = 1000*timeoutSec;
         }
 
+        requestHeaders.put("ET-Client-Name", SiriHttpUtils.getUniqueETClientName("-VM"));
     }
 
     @Override
@@ -68,7 +73,7 @@ public Siri getUpdates() {
             creating = System.currentTimeMillis()-t1;
             t1 = System.currentTimeMillis();
 
-            InputStream is = SiriHttpUtils.postData(url, vmServiceRequest, timeout);
+            InputStream is = SiriHttpUtils.postData(url, vmServiceRequest, timeout, requestHeaders);
             if (is != null) {
                 // Decode message
                 fetching = System.currentTimeMillis()-t1;

src/ext/java/org/opentripplanner/ext/transmodelapi/TransmodelGraphQLPlanner.java

@@ -88,7 +88,7 @@ public Map<String, Object> plan(DataFetchingEnvironment environment) {
 
 
             if (request.modes.isTransit()) {
-                RaptorRouter raptorRouter = new RaptorRouter(request, router.graph.transitLayer);
+                RaptorRouter raptorRouter = new RaptorRouter(request, router.graph.getRealtimeTransitLayer());
                 itineraries.addAll(raptorRouter.route());
             }