Skip to content

Commit b6d1058

Browse files
landonreedlandonreed
committed
feat(merge-feeds): add MTC merge feeds strategy
resolves #185
1 parent cf7208c commit b6d1058

17 files changed

+814
-265
lines changed

‎src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,14 @@
2828
import java.util.Date;
2929
import java.util.HashSet;
3030
import java.util.Set;
31-
import java.util.stream.Collectors;
32-
import java.util.stream.Stream;
3331

3432
import static com.conveyal.datatools.common.utils.S3Utils.downloadFromS3;
3533
import static com.conveyal.datatools.common.utils.SparkUtils.copyRequestStreamIntoFile;
3634
import static com.conveyal.datatools.common.utils.SparkUtils.downloadFile;
3735
import static com.conveyal.datatools.common.utils.SparkUtils.formatJobMessage;
3836
import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
3937
import static com.conveyal.datatools.manager.controllers.api.FeedSourceController.checkFeedSourcePermissions;
38+
import static com.conveyal.datatools.manager.jobs.MergeFeedsType.MTC;
4039
import static spark.Spark.delete;
4140
import static spark.Spark.get;
4241
import static spark.Spark.post;
@@ -289,7 +288,7 @@ else if (!v.feedSourceId.equals(feedSourceId)) {
289288
if (versionIds.length != 2) {
290289
logMessageAndHalt(req, 400, "Merging more than two versions is not currently supported.");
291290
}
292-
MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(userProfile.getUser_id(), versions, "merged", MergeFeedsJob.MergeType.FEED_VERSIONS);
291+
MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(userProfile.getUser_id(), versions, "merged", MTC);
293292
DataManager.heavyExecutor.execute(mergeFeedsJob);
294293
return SparkUtils.formatJobMessage(mergeFeedsJob.jobId, "Merging feed versions...");
295294
}

‎src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@
55
import com.conveyal.datatools.manager.auth.Auth0UserProfile;
66
import com.conveyal.datatools.manager.jobs.FetchProjectFeedsJob;
77
import com.conveyal.datatools.manager.jobs.MakePublicJob;
8-
import com.conveyal.datatools.manager.jobs.MergeProjectFeedsJob;
8+
import com.conveyal.datatools.manager.jobs.MergeFeedsJob;
99
import com.conveyal.datatools.manager.models.FeedDownloadToken;
10+
import com.conveyal.datatools.manager.models.FeedSource;
1011
import com.conveyal.datatools.manager.models.FeedVersion;
1112
import com.conveyal.datatools.manager.models.JsonViews;
1213
import com.conveyal.datatools.manager.models.Project;
@@ -20,13 +21,16 @@
2021
import spark.Response;
2122

2223
import java.util.Collection;
24+
import java.util.HashSet;
25+
import java.util.Set;
2326
import java.util.stream.Collectors;
2427

2528
import static com.conveyal.datatools.common.utils.S3Utils.downloadFromS3;
2629
import static com.conveyal.datatools.common.utils.SparkUtils.downloadFile;
2730
import static com.conveyal.datatools.common.utils.SparkUtils.formatJobMessage;
2831
import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
2932
import static com.conveyal.datatools.manager.DataManager.publicPath;
33+
import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL;
3034
import static spark.Spark.delete;
3135
import static spark.Spark.get;
3236
import static spark.Spark.post;
@@ -220,10 +224,24 @@ private static String downloadMergedFeed(Request req, Response res) {
220224
Project project = requestProjectById(req, "view");
221225
Auth0UserProfile userProfile = req.attribute("user");
222226
// TODO: make this an authenticated call?
223-
MergeProjectFeedsJob mergeProjectFeedsJob = new MergeProjectFeedsJob(project, userProfile.getUser_id());
224-
DataManager.heavyExecutor.execute(mergeProjectFeedsJob);
227+
Set<FeedVersion> feedVersions = new HashSet<>();
228+
// Get latest version for each feed source in project
229+
Collection<FeedSource> feedSources = project.retrieveProjectFeedSources();
230+
for (FeedSource fs : feedSources) {
231+
// check if feed version exists
232+
FeedVersion version = fs.retrieveLatest();
233+
if (version == null) {
234+
LOG.warn("Skipping {} because it has no feed versions", fs.name);
235+
continue;
236+
}
237+
// modify feed version to use prepended feed id
238+
LOG.info("Adding {} feed to merged zip", fs.name);
239+
feedVersions.add(version);
240+
}
241+
MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(userProfile.getUser_id(), feedVersions, project.id, REGIONAL);
242+
DataManager.heavyExecutor.execute(mergeFeedsJob);
225243
// Return job ID to requester for monitoring job status.
226-
return formatJobMessage(mergeProjectFeedsJob.jobId, "Merge operation is processing.");
244+
return formatJobMessage(mergeFeedsJob.jobId, "Merge operation is processing.");
227245
}
228246

229247
/**

‎src/main/java/com/conveyal/datatools/manager/gtfsplus/CalendarAttribute.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.conveyal.datatools.manager.gtfsplus;
2+
3+
import com.conveyal.gtfs.model.Entity;
4+
5+
import java.sql.PreparedStatement;
6+
import java.sql.SQLException;
7+
8+
public class CalendarAttribute extends Entity {
9+
10+
private static final long serialVersionUID = 1L;
11+
12+
public String service_id;
13+
public String service_description;
14+
15+
// TODO
16+
@Override
17+
public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
18+
19+
}
20+
}

‎src/main/java/com/conveyal/datatools/manager/gtfsplus/Direction.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package com.conveyal.datatools.manager.gtfsplus;
2+
3+
import com.conveyal.gtfs.model.Entity;
4+
5+
import java.sql.PreparedStatement;
6+
import java.sql.SQLException;
7+
8+
public class Direction extends Entity {
9+
10+
private static final long serialVersionUID = 1L;
11+
12+
public String route_id;
13+
public int direction_id;
14+
public String direction;
15+
16+
17+
// TODO
18+
@Override
19+
public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
20+
21+
}
22+
}

‎src/main/java/com/conveyal/datatools/manager/gtfsplus/FareRiderCategory.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package com.conveyal.datatools.manager.gtfsplus;
2+
3+
import com.conveyal.gtfs.model.Entity;
4+
5+
import java.sql.PreparedStatement;
6+
import java.sql.SQLException;
7+
import java.time.LocalDate;
8+
9+
public class FareRiderCategory extends Entity {
10+
11+
private static final long serialVersionUID = 1L;
12+
13+
public String fare_id;
14+
public int rider_category_id;
15+
public double price;
16+
public LocalDate expiration_date;
17+
public LocalDate commencement_date;
18+
19+
// TODO
20+
@Override
21+
public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
22+
23+
}
24+
}

‎src/main/java/com/conveyal/datatools/manager/gtfsplus/FareZoneAttribute.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.conveyal.datatools.manager.gtfsplus;
2+
3+
import com.conveyal.gtfs.model.Entity;
4+
5+
import java.sql.PreparedStatement;
6+
import java.sql.SQLException;
7+
8+
public class FareZoneAttribute extends Entity {
9+
10+
private static final long serialVersionUID = 1L;
11+
12+
public String zone_id;
13+
public String zone_name;
14+
15+
// TODO
16+
@Override
17+
public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
18+
19+
}
20+
}

‎src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusTable.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package com.conveyal.datatools.manager.gtfsplus;
2+
3+
import com.conveyal.gtfs.loader.DateField;
4+
import com.conveyal.gtfs.loader.DoubleField;
5+
import com.conveyal.gtfs.loader.IntegerField;
6+
import com.conveyal.gtfs.loader.ShortField;
7+
import com.conveyal.gtfs.loader.StringField;
8+
import com.conveyal.gtfs.loader.Table;
9+
10+
import static com.conveyal.gtfs.loader.Requirement.OPTIONAL;
11+
import static com.conveyal.gtfs.loader.Requirement.PROPRIETARY;
12+
import static com.conveyal.gtfs.loader.Requirement.REQUIRED;
13+
14+
public class GtfsPlusTable {
15+
public static final Table REALTIME_ROUTES = new Table("realtime_routes", RealtimeRoute.class, PROPRIETARY,
16+
new StringField("route_id", REQUIRED).isReferenceTo(Table.ROUTES),
17+
new ShortField("realtime_enabled", REQUIRED, 1),
18+
new StringField("realtime_routename", REQUIRED),
19+
new StringField("realtime_routecode", REQUIRED)
20+
);
21+
22+
public static final Table REALTIME_STOPS = new Table("realtime_stops", RealtimeStop.class, PROPRIETARY,
23+
new StringField("trip_id", REQUIRED).isReferenceTo(Table.TRIPS),
24+
new StringField("stop_id", REQUIRED).isReferenceTo(Table.STOPS),
25+
new StringField("realtime_stop_id", REQUIRED)
26+
).keyFieldIsNotUnique();
27+
28+
public static final Table DIRECTIONS = new Table("directions", Direction.class, PROPRIETARY,
29+
new StringField("route_id", REQUIRED).isReferenceTo(Table.ROUTES),
30+
new ShortField("direction_id", REQUIRED, 1),
31+
new StringField("direction", REQUIRED)
32+
).keyFieldIsNotUnique();
33+
34+
public static final Table REALTIME_TRIPS = new Table("realtime_trips", RealtimeTrip.class, PROPRIETARY,
35+
new StringField("trip_id", REQUIRED).isReferenceTo(Table.TRIPS),
36+
new StringField("realtime_trip_id", REQUIRED)
37+
);
38+
39+
public static final Table STOP_ATTRIBUTES = new Table("stop_attributes", StopAttribute.class, PROPRIETARY,
40+
new StringField("stop_id", REQUIRED).isReferenceTo(Table.STOPS),
41+
new ShortField("accessibility_id", REQUIRED, 8),
42+
new StringField("cardinal_direction", OPTIONAL),
43+
new StringField("relative_position", OPTIONAL),
44+
new StringField("stop_city", REQUIRED)
45+
);
46+
47+
public static final Table TIMEPOINTS = new Table("timepoints", TimePoint.class, PROPRIETARY,
48+
new StringField("trip_id", REQUIRED).isReferenceTo(Table.TRIPS),
49+
new StringField("stop_id", REQUIRED).isReferenceTo(Table.STOPS)
50+
).keyFieldIsNotUnique();
51+
52+
public static final Table RIDER_CATEGORIES = new Table("rider_categories", RiderCategory.class, PROPRIETARY,
53+
new IntegerField("rider_category_id", REQUIRED, 1, 25),
54+
new StringField("rider_category_description", REQUIRED)
55+
);
56+
57+
public static final Table FARE_RIDER_CATEGORIES = new Table("fare_rider_categories", FareRiderCategory.class, PROPRIETARY,
58+
new StringField("fare_id", REQUIRED),
59+
new IntegerField("rider_category_id", REQUIRED, 2, 25).isReferenceTo(RIDER_CATEGORIES),
60+
new DoubleField("price", REQUIRED, 0, Double.MAX_VALUE, 2),
61+
new DateField("expiration_date", OPTIONAL),
62+
new DateField("commencement_date", OPTIONAL)
63+
).keyFieldIsNotUnique();
64+
65+
public static final Table CALENDAR_ATTRIBUTES = new Table("calendar_attributes", CalendarAttribute.class, PROPRIETARY,
66+
new StringField("service_id", REQUIRED).isReferenceTo(Table.CALENDAR),
67+
new StringField("service_description", REQUIRED)
68+
);
69+
70+
public static final Table FAREZONE_ATTRIBUTES = new Table("farezone_attributes", FareZoneAttribute.class, PROPRIETARY,
71+
new StringField("zone_id", REQUIRED),
72+
new StringField("zone_name", REQUIRED)
73+
);
74+
75+
public static final Table[] tables = new Table[] {
76+
REALTIME_ROUTES,
77+
REALTIME_STOPS,
78+
REALTIME_TRIPS,
79+
DIRECTIONS,
80+
STOP_ATTRIBUTES,
81+
TIMEPOINTS,
82+
RIDER_CATEGORIES,
83+
FARE_RIDER_CATEGORIES,
84+
CALENDAR_ATTRIBUTES,
85+
FAREZONE_ATTRIBUTES
86+
};
87+
}

‎src/main/java/com/conveyal/datatools/manager/gtfsplus/RealtimeRoute.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package com.conveyal.datatools.manager.gtfsplus;
2+
3+
import com.conveyal.gtfs.model.Entity;
4+
5+
import java.sql.PreparedStatement;
6+
import java.sql.SQLException;
7+
8+
public class RealtimeRoute extends Entity {
9+
private static final long serialVersionUID = 1L;
10+
11+
public String route_id;
12+
public int realtime_enabled;
13+
public String realtime_routename;
14+
public String realtime_routecode;
15+
16+
// TODO
17+
@Override
18+
public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
19+
20+
}
21+
}

‎src/main/java/com/conveyal/datatools/manager/gtfsplus/RealtimeStop.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package com.conveyal.datatools.manager.gtfsplus;
2+
3+
import com.conveyal.gtfs.model.Entity;
4+
5+
import java.sql.PreparedStatement;
6+
import java.sql.SQLException;
7+
8+
public class RealtimeStop extends Entity {
9+
10+
private static final long serialVersionUID = 1L;
11+
12+
public String trip_id;
13+
public String stop_id;
14+
public String realtime_stop_id;
15+
16+
// TODO
17+
@Override
18+
public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
19+
20+
}
21+
}

‎src/main/java/com/conveyal/datatools/manager/gtfsplus/RealtimeTrip.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.conveyal.datatools.manager.gtfsplus;
2+
3+
import com.conveyal.gtfs.model.Entity;
4+
5+
import java.sql.PreparedStatement;
6+
import java.sql.SQLException;
7+
8+
public class RealtimeTrip extends Entity {
9+
10+
private static final long serialVersionUID = 1L;
11+
12+
public String trip_id;
13+
public String realtime_trip_id;
14+
15+
// TODO
16+
@Override
17+
public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
18+
19+
}
20+
}

0 commit comments

Comments
 (0)