Skip to content

Commit

Permalink
initial work on analyst qa framework (#2107).
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattwigway
mattwigway committed Aug 20, 2015
1 parent ea27bc6 commit b70c824
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 96 deletions.
219 changes: 126 additions & 93 deletions src/main/java/org/opentripplanner/analyst/RepeatedRaptorComparison.java
View file
Expand Up @@ -2,33 +2,30 @@


import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Coordinate;
import org.joda.time.LocalDate; import org.joda.time.LocalDate;
import org.mapdb.BTreeMap; import org.mapdb.*;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.Serializer;
import org.opentripplanner.analyst.cluster.ResultEnvelope; import org.opentripplanner.analyst.cluster.ResultEnvelope;
import org.opentripplanner.analyst.cluster.TaskStatistics; import org.opentripplanner.analyst.cluster.TaskStatistics;
import org.opentripplanner.api.parameter.QualifiedModeSet; import org.opentripplanner.api.parameter.QualifiedModeSet;
import org.opentripplanner.common.MavenVersion; import org.opentripplanner.common.MavenVersion;
import org.opentripplanner.graph_builder.GraphBuilder; import org.opentripplanner.graph_builder.GraphBuilder;
import org.opentripplanner.profile.ProfileRequest; import org.opentripplanner.profile.ProfileRequest;
import org.opentripplanner.profile.RaptorWorker;
import org.opentripplanner.profile.RaptorWorkerData; import org.opentripplanner.profile.RaptorWorkerData;
import org.opentripplanner.profile.RepeatedRaptorProfileRouter; import org.opentripplanner.profile.RepeatedRaptorProfileRouter;
import org.opentripplanner.routing.core.TraverseModeSet;
import org.opentripplanner.routing.graph.Graph; import org.opentripplanner.routing.graph.Graph;
import org.opentripplanner.routing.graph.Vertex; import org.opentripplanner.routing.graph.Vertex;
import org.opentripplanner.routing.impl.DefaultStreetVertexIndexFactory; import org.opentripplanner.routing.impl.DefaultStreetVertexIndexFactory;
import org.opentripplanner.routing.vertextype.OsmVertex;
import org.opentripplanner.standalone.CommandLineParameters; import org.opentripplanner.standalone.CommandLineParameters;
import org.opentripplanner.streets.Histogram; import org.opentripplanner.streets.Histogram;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


import java.io.File; import java.io.File;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.IntStream;


/** /**
* Compare results from Repeated RAPTOR runs. This is used to detect algorithmic changes that have * Compare results from Repeated RAPTOR runs. This is used to detect algorithmic changes that have
Expand All @@ -43,6 +40,7 @@
* *
*/ */
public class RepeatedRaptorComparison { public class RepeatedRaptorComparison {
private static final Logger LOG = LoggerFactory.getLogger(RepeatedRaptorComparison.class);


// amounts to offset origins from vertices to test linking // amounts to offset origins from vertices to test linking
public static final double OFFSET_X = 1e-4, OFFSET_Y = 1e-4; public static final double OFFSET_X = 1e-4, OFFSET_Y = 1e-4;
Expand All @@ -58,15 +56,15 @@ public static void main (String... args) {
Graph graph = buildGraph(graphDir); Graph graph = buildGraph(graphDir);


DB comparisonDb = null; DB comparisonDb = null;
BTreeMap<String, ResultEnvelope> comparison = null; BTreeMap<Fun.Tuple3<String, String, ResultEnvelope.Which>, Integer> comparison = null;


// open the comparison file, if we have one. // open the comparison file, if we have one.
if (args.length > 1) { if (args.length > 1) {
comparisonDb = DBMaker.newFileDB(new File(args[1])) comparisonDb = DBMaker.newFileDB(new File(args[1]))
.readOnly() .readOnly()
.transactionDisable() .transactionDisable()
.closeOnJvmShutdown() .closeOnJvmShutdown()
.cacheSize(48) .cacheSize(24)
.asyncWriteEnable() .asyncWriteEnable()
.make(); .make();


Expand All @@ -81,45 +79,44 @@ public static void main (String... args) {
.closeOnJvmShutdown() .closeOnJvmShutdown()
.make(); .make();


final BTreeMap<String, ResultEnvelope> output = outputDb.createTreeMap("results") final BTreeMap<Fun.Tuple3<String, String, ResultEnvelope.Which>, Integer> output = outputDb.createTreeMap("results")
.valueSerializer(Serializer.JAVA) .valueSerializer(Serializer.JAVA)
.makeStringMap(); .makeOrGet();


// if we have a comparison file, get the vertices from it. Otherwise choose some randomly. // if we have a comparison file, get the pointset from it. Otherwise choose some randomly.
Collection<String> vertexLabels; Collection<String> vertexLabels;
PointSet pset; PointSet pset;


if (comparison != null) { if (comparison != null) {
vertexLabels = comparison.keySet();
// clooge, pointset is stored in its own map in db. // clooge, pointset is stored in its own map in db.
pset = comparisonDb.<String, PointSet>getTreeMap("pointset").get("pointset"); pset = comparisonDb.<String, PointSet>getTreeMap("pointset").get("pointset");
} }
else { else {
// choose some vertices // choose some vertices
List<String> labelList = graph.getVertices().stream() List<Vertex> vertices = graph.getVertices().stream()
.map(Vertex::getLabel) // only use OSM nodes, because they are stable. e.g. splittervertices may not have stable identifiers between builds.
// only use OSM nodes, because they are stable. e.g. splits may change name. .filter(v -> v.getLabel().startsWith("osm:node:"))
.filter(s -> s.startsWith("osm:node:")) .limit(1000)
.collect(Collectors.toList()); .collect(Collectors.toList());


if (labelList.size() > 10000) {
Collections.shuffle(labelList);
vertexLabels = labelList.stream().limit(10000).collect(Collectors.toList());
}
else {
vertexLabels = labelList;
}

// make a pointset // make a pointset
pset = PointSet.regularGrid(graph.getExtent(), 100); pset = new PointSet(vertices.size());
int featIdx = 0;
for (Vertex v : vertices) {
PointFeature pf = new PointFeature();
pf.setId(v.getLabel());
pf.setLat(v.getLat() + OFFSET_Y);
pf.setLon(v.getLon() + OFFSET_X);
pset.addFeature(pf, featIdx++);
}


outputDb.createTreeMap("pointset") outputDb.createTreeMap("pointset")
.<String, PointSet>make().put("pointset", pset); .<String, PointSet>make().put("pointset", pset);
} }


SampleSet ss = new SampleSet(pset, graph.getSampleFactory()); SampleSet ss = new SampleSet(pset, graph.getSampleFactory());


final BTreeMap<String, ResultEnvelope> comparisonResults = comparison; final BTreeMap<Fun.Tuple3<String, String, ResultEnvelope.Which>, Integer> comparisonResults = comparison;


org.opentripplanner.streets.Histogram bestCaseHisto = new Histogram("Best case"); org.opentripplanner.streets.Histogram bestCaseHisto = new Histogram("Best case");
org.opentripplanner.streets.Histogram avgCaseHisto = new Histogram("Average"); org.opentripplanner.streets.Histogram avgCaseHisto = new Histogram("Average");
Expand All @@ -128,70 +125,115 @@ public static void main (String... args) {
ProfileRequest template = new ProfileRequest(); ProfileRequest template = new ProfileRequest();
template.accessModes = new QualifiedModeSet("WALK"); template.accessModes = new QualifiedModeSet("WALK");
template.analyst = true; template.analyst = true;
// This means results will vary slightly, which is why we plot a histogram when we're done.
//req.boardingAssumption = RaptorWorkerTimetable.BoardingAssumption.RANDOM;
template.maxWalkTime = 20 * 60; template.maxWalkTime = 20 * 60;
template.walkSpeed = 1.3f;
template.fromTime = 7 * 3600; template.fromTime = 7 * 3600;
template.toTime = 9 * 3600; template.toTime = 9 * 3600;


template.date = new LocalDate(2015, 8, 4); template.date = new LocalDate(2015, 8, 4);


RaptorWorkerData data = RepeatedRaptorProfileRouter.getRaptorWorkerData(template, graph, ss, new TaskStatistics()); RaptorWorkerData data = RepeatedRaptorProfileRouter.getRaptorWorkerData(template, graph, ss, new TaskStatistics());


ThreadPoolExecutor executor = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors() * 2,
10, TimeUnit.SECONDS, new ArrayBlockingQueue<>(10000), new ThreadPoolExecutor.CallerRunsPolicy());

// do the computation and comparison // do the computation and comparison
vertexLabels.stream() IntStream.range(0, pset.featureCount()).parallel()
.forEach(label -> { .forEach(idx -> {
executor.execute(() -> {
OsmVertex v = (OsmVertex) graph.getVertex(label); if (idx % 100 == 0)

System.out.println(idx + " points complete");
// offset a deterministic amount to test linking
Coordinate coord = new Coordinate(v.getX() + OFFSET_X, v.getY() + OFFSET_Y); Coordinate coord = pset.getCoordinate(idx);

String origin = pset.getFeature(idx).getId();
ProfileRequest req;
try { ProfileRequest req;
req = template.clone(); try {
} catch (CloneNotSupportedException e) { req = template.clone();
/* can't happen */ } catch (CloneNotSupportedException e) {
throw new RuntimeException(e); /* can't happen */
} throw new RuntimeException(e);

}
req.maxWalkTime = 20 * 60;
req.fromLat = req.toLat = coord.y; req.maxWalkTime = 20 * 60;
req.fromLon = req.toLon = coord.x; req.fromLat = req.toLat = coord.y;

req.fromLon = req.toLon = coord.x;
RepeatedRaptorProfileRouter rrpr = new RepeatedRaptorProfileRouter( // 7 to 9 AM
graph, req, ss); req.fromTime = 7 * 3600;
rrpr.raptorWorkerData = data; req.toTime = 9 * 3600;
rrpr.route(); req.transitModes = new TraverseModeSet("TRANSIT");


ResultSet.RangeSet results = rrpr.makeResults(true, true, false); RepeatedRaptorProfileRouter rrpr = new RepeatedRaptorProfileRouter(graph, req, ss);
ResultEnvelope env = new ResultEnvelope(); rrpr.raptorWorkerData = data;
env.bestCase = results.min;
env.worstCase = results.max; try {
env.avgCase = results.avg; rrpr.route();
env.id = label; } catch (Exception e) {
env.profile = true; LOG.error("Exception during routing", e);

return;
if (comparisonResults != null) { }
ResultEnvelope compare = comparisonResults.get(label);
compare(env.bestCase, compare.bestCase, bestCaseHisto); ResultSet.RangeSet results = rrpr.makeResults(true, false, false);
compare(env.worstCase, compare.worstCase, worstCaseHisto);
compare(env.avgCase, compare.avgCase, avgCaseHisto); for (ResultEnvelope.Which which : new ResultEnvelope.Which[] {
} ResultEnvelope.Which.BEST_CASE, ResultEnvelope.Which.AVERAGE,

ResultEnvelope.Which.WORST_CASE }) {
output.put(env.id, env); Histogram histogram;
}); ResultSet resultSet;
});

switch (which) {
// wait for execution to complete case BEST_CASE:
try { histogram = bestCaseHisto;
executor.awaitTermination(10, TimeUnit.DAYS); resultSet = results.min;
} catch (InterruptedException e) { break;
System.err.println("interrupted"); case WORST_CASE:
} histogram = worstCaseHisto;
resultSet = results.max;
break;
case AVERAGE:
histogram = avgCaseHisto;
resultSet = results.avg;
break;
default:
histogram = null;
resultSet = null;
}

// now that we have the proper histogram and result set, save them and do the
// comparison.
for (int i = 0; i < resultSet.times.length; i++) {
int time = resultSet.times[i];
// TODO this is creating a PointFeature obj to hold the id at each call
// Cache?
String dest = pset.getFeature(i).getId();

Fun.Tuple3<String, String, ResultEnvelope.Which> key = new Fun.Tuple3<>(
origin, dest, which);
output.put(key, time);

if (time < 0) {
LOG.error("Path from {} to {} has negative time {}", origin, dest,
time);
}

if (comparisonResults != null) {
int time0 = comparisonResults.get(key);

int deltaMinutes;

if (time0 == RaptorWorker.UNREACHED && time != RaptorWorker.UNREACHED)
deltaMinutes = (time / 60) - 120;
else if (time == RaptorWorker.UNREACHED
&& time0 != RaptorWorker.UNREACHED)
deltaMinutes = 120 - (time0 / 60);
else
deltaMinutes = (time - time0) / 60;

// histograms are not threadsafe
synchronized (histogram) {
histogram.add(deltaMinutes);
}
}
}
}
});


output.close(); output.close();
if (comparisonDb != null) { if (comparisonDb != null) {
Expand Down Expand Up @@ -220,13 +262,4 @@ private static Graph buildGraph(File directory) {
graph.index.clusterStopsAsNeeded(); graph.index.clusterStopsAsNeeded();
return graph; return graph;
} }

private static void compare(ResultSet current, ResultSet compare, Histogram histogram) {
// histograms aren't threadsafe
synchronized (histogram) {
for (int i = 0; i < current.times.length; i++) {
histogram.add((int) Math.round((current.times[i] - compare.times[i]) / 60d));
}
}
}
} }
2 changes: 1 addition & 1 deletion src/main/java/org/opentripplanner/profile/ProfileRequest.java
View file
Expand Up @@ -82,7 +82,7 @@ public class ProfileRequest implements Serializable, Cloneable {
public boolean analyst = false; public boolean analyst = false;


/** What assumption should be used when boarding frequency vehicles? */ /** What assumption should be used when boarding frequency vehicles? */
public RaptorWorkerTimetable.BoardingAssumption boardingAssumption = RaptorWorkerTimetable.BoardingAssumption.WORST_CASE; public RaptorWorkerTimetable.BoardingAssumption boardingAssumption = RaptorWorkerTimetable.BoardingAssumption.RANDOM;


/* The relative importance of different factors when biking */ /* The relative importance of different factors when biking */
/** The relative importance of maximizing safety when cycling */ /** The relative importance of maximizing safety when cycling */
Expand Down
19 changes: 17 additions & 2 deletions src/main/java/org/opentripplanner/streets/Histogram.java
View file
Expand Up @@ -4,6 +4,9 @@
import gnu.trove.iterator.TIntIntIterator; import gnu.trove.iterator.TIntIntIterator;
import gnu.trove.map.TIntIntMap; import gnu.trove.map.TIntIntMap;
import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.map.hash.TIntIntHashMap;
import org.apache.commons.math3.random.MersenneTwister;

import java.util.stream.IntStream;


/** /**
* For design and debugging purposes, a simple class that tracks the frequency of different numbers. * For design and debugging purposes, a simple class that tracks the frequency of different numbers.
Expand Down Expand Up @@ -83,7 +86,7 @@ public void displayHorizontal () {
for (int i = 0; i < 30; i++) { for (int i = 0; i < 30; i++) {
StringBuilder row = new StringBuilder(maxBin - minBin + 1); StringBuilder row = new StringBuilder(maxBin - minBin + 1);


int minValToDisplayThisRow = (int) (i / vscale); int minValToDisplayThisRow = (int) ((30 - i) / vscale);
for (int j = minBin; j <= maxBin; j++) { for (int j = minBin; j <= maxBin; j++) {
if (bins.get(j) > minValToDisplayThisRow) if (bins.get(j) > minValToDisplayThisRow)
row.append('#'); row.append('#');
Expand Down Expand Up @@ -111,7 +114,7 @@ public void displayHorizontal () {
String start = new Integer(minBin).toString(); String start = new Integer(minBin).toString();
row.replace(0, start.length(), start); row.replace(0, start.length(), start);
String end = new Integer(maxBin).toString(); String end = new Integer(maxBin).toString();
row.replace(row.length() - end.length(), end.length(), end); row.replace(row.length() - end.length(), row.length(), end);
System.out.println(row); System.out.println(row);
} }


Expand All @@ -125,4 +128,16 @@ public int mean() {


return (int) (sum / count); return (int) (sum / count);
} }

public static void main (String... args) {
System.out.println("Testing histogram store with normal distribution, mean 0");
Histogram h = new Histogram("Normal");

MersenneTwister mt = new MersenneTwister();

IntStream.range(0, 1000000).map(i -> (int) Math.round(mt.nextGaussian() * 20 + 2.5)).forEach(h::add);

h.displayHorizontal();
System.out.println("mean: " + h.mean());
}
} }

0 comments on commit b70c824

Please sign in to comment.