Skip to content

Commit

Permalink
Unse @transactional on methods (doesn't seem to work).
Browse files Browse the repository at this point in the history
  • Loading branch information
Matt Conway committed Sep 18, 2012
1 parent 4398adb commit bb43cb5
Show file tree
Hide file tree
Showing 6 changed files with 283 additions and 270 deletions.
1 change: 0 additions & 1 deletion app/jobs/UpdateGtfs.java
View file
Expand Up @@ -32,7 +32,6 @@ the License, or (props, at your option) any later version.
// the reason for the odd time is because it seems likely that a good number of people
// start cron jobs on the hour on AWS; this will fall between those.
@On("0 37 7 * * ?")
@NoTransaction
public class UpdateGtfs extends Job {
/** Is this job currently running? This prevents trying to update GTFS twice at the same time */
private static boolean running = false;
Expand Down
3 changes: 3 additions & 0 deletions app/updaters/DeploymentPlanGeneratorHook.java
View file
Expand Up @@ -21,6 +21,7 @@ the License, or (props, at your option) any later version.

import play.Play;
import play.db.jpa.JPAPlugin;
import play.db.jpa.Transactional;

import deployment.DeploymentPlan;

Expand All @@ -34,6 +35,8 @@ the License, or (props, at your option) any later version.
public class DeploymentPlanGeneratorHook implements UpdaterHook {

@Override
// this is not read only because rebuild scheduling needs a read-write transaction.
@Transactional(readOnly=false)
public void update(Set<MetroArea> areas) {
JPAPlugin.startTx(true);

Expand Down
244 changes: 128 additions & 116 deletions app/updaters/GtfsDataExchangeUpdater.java
View file
Expand Up @@ -28,6 +28,7 @@ the License, or (props, at your option) any later version.
import play.db.jpa.JPA;
import play.db.jpa.JPAPlugin;
import play.db.jpa.NoTransaction;
import play.db.jpa.Transactional;
import play.exceptions.JPAException;
import play.libs.WS;
import play.libs.WS.HttpResponse;
Expand Down Expand Up @@ -68,128 +69,139 @@ public Set<MetroArea> update (FeedStorer storer) {
JsonObject feed;

for (JsonElement rawFeed : data) {
// this doesn't seem to cause a problem if there is no active transaction
JPAPlugin.closeTx(true);
JPAPlugin.startTx(false);

feed = rawFeed.getAsJsonObject();

String dataExchangeId = feed.get("dataexchange_id").getAsString();

if (dataExchangeId == null || dataExchangeId.equals(""))
continue;

// ORDER BY ...: so failed feeds aren't continuously refetched.
GtfsFeed originalFeed = GtfsFeed.find("dataExchangeId = ? AND supersededBy IS NULL ORDER BY dateUpdated DESC",
dataExchangeId).first();

// convert to ms
long lastUpdated = ((long) feed.get("date_last_updated").getAsDouble()) * 1000L;

// do we need to fetch?
if (originalFeed != null) {
// difference of less than 2000ms: ignore
if ((lastUpdated - originalFeed.dateUpdated.getTime()) < 2000) {
continue;
}
}

// get the data file URL
res = WS.url("http://www.gtfs-data-exchange.com/agency/" +
dataExchangeId + "/json").get();
status = res.getStatus();
if (status != 200) {
Logger.error("Error fetching agency %s, status %s", dataExchangeId, status);
continue;
}

JsonObject agency = res.getJson().getAsJsonObject();
JsonArray files = agency.get("data").getAsJsonObject()
.get("datafiles").getAsJsonArray();
JsonObject firstFile = files.get(0).getAsJsonObject();
String url = firstFile.get("file_url").getAsString();

// Download the feed
String feedId = storer.storeFeed(url);
if (feedId == null) {
Logger.error("Could not retrieve feed %s", url);
// feed will be redownloaded on next attempt
continue;
}

GtfsFeed newFeed;
boolean isNew;
// copy over all the data.
if (originalFeed != null) {
isNew = false;
newFeed = originalFeed.clone();
}
else {
newFeed = new GtfsFeed();
newFeed.note = "new feed";
isNew = true;
}

// update all fields
FeedUtils.copyFromJson(feed, newFeed);
newFeed.downloadUrl = url;
newFeed.storedId = feedId;

// Calculate feed stats
File feedFile = storer.getFeed(feedId);

FeedStatsCalculator stats;

try {
stats = new FeedStatsCalculator(feedFile);
calculateAndSave(feed, storer, updated);
} catch (Exception e) {
// TODO be more descriptive
Logger.error("Error calculating feed stats for feed %s", url);
e.printStackTrace();

// still save it in the DB
newFeed.status = FeedParseStatus.FAILED;
newFeed.save();

if (originalFeed != null) {
originalFeed.supersededBy = newFeed;
originalFeed.save();
}

JPAPlugin.closeTx(false);

continue;
}

storer.releaseFeed(feedId);

// save the stats
stats.apply(newFeed);
newFeed.status = FeedParseStatus.SUCCESSFUL;
newFeed.save();

// if it's a new feed, find an agency.
if (isNew) {
if (!newFeed.findAgency())
newFeed.review = ReviewType.NO_AGENCY;

newFeed.save();
Logger.error("Error fetching/saving feed %s",
feed.get("dataexchange_id").getAsString());
}

for (NtdAgency ntd : newFeed.getEnabledAgencies()) {
for (MetroArea metro : ntd.getEnabledMetroAreas()) {
updated.add(metro);
}
}

if (originalFeed != null) {
originalFeed.supersededBy = newFeed;
originalFeed.save();
}

// https://groups.google.com/forum/?fromgroups=#!topic/play-framework/qaIaMOIjpMM
JPAPlugin.closeTx(false);
}

return updated;
}

/**
* Download the GTFS Data Exchange feed described by feed and save it to the DB and to
* the storer named in storer, putting any updated metro areas into updated (which will be
* modified)
* @param feed
* @param storer
* @param updated
*/
@Transactional(readOnly=false)
private void calculateAndSave(JsonObject feed, FeedStorer storer, Set<MetroArea> updated) {
String dataExchangeId = feed.get("dataexchange_id").getAsString();

if (dataExchangeId == null || dataExchangeId.equals(""))
return;

// ORDER BY ...: so failed feeds aren't continuously refetched.
GtfsFeed originalFeed = GtfsFeed.find("dataExchangeId = ? AND supersededBy IS NULL ORDER BY dateUpdated DESC",
dataExchangeId).first();

// convert to ms
long lastUpdated = ((long) feed.get("date_last_updated").getAsDouble()) * 1000L;

// do we need to fetch?
if (originalFeed != null) {
// difference of less than 2000ms: ignore
if ((lastUpdated - originalFeed.dateUpdated.getTime()) < 2000) {
return;
}
}

// get the data file URL
HttpResponse res = WS.url("http://www.gtfs-data-exchange.com/agency/" +
dataExchangeId + "/json").get();
if (!res.success()) {
Logger.error("Error fetching agency %s, status %s", dataExchangeId, res.getStatus());
return;
}

JsonObject agency = res.getJson().getAsJsonObject();
JsonArray files = agency.get("data").getAsJsonObject()
.get("datafiles").getAsJsonArray();
JsonObject firstFile = files.get(0).getAsJsonObject();
String url = firstFile.get("file_url").getAsString();

// Download the feed
String feedId = storer.storeFeed(url);
if (feedId == null) {
Logger.error("Could not retrieve feed %s", url);
// feed will be redownloaded on next attempt
return;
}

GtfsFeed newFeed;
boolean isNew;
// copy over all the data.
if (originalFeed != null) {
isNew = false;
newFeed = originalFeed.clone();
}
else {
newFeed = new GtfsFeed();
newFeed.note = "new feed";
isNew = true;
}

// update all fields
FeedUtils.copyFromJson(feed, newFeed);
newFeed.downloadUrl = url;
newFeed.storedId = feedId;

// Calculate feed stats
File feedFile = storer.getFeed(feedId);

FeedStatsCalculator stats;
try {
stats = new FeedStatsCalculator(feedFile);
} catch (Exception e) {
// TODO be more descriptive
Logger.error("Error calculating feed stats for feed %s", url);
e.printStackTrace();

// still save it in the DB
newFeed.status = FeedParseStatus.FAILED;
newFeed.save();

if (originalFeed != null) {
originalFeed.supersededBy = newFeed;
originalFeed.save();
}

JPAPlugin.closeTx(false);

return;
}

storer.releaseFeed(feedId);

// save the stats
stats.apply(newFeed);
newFeed.status = FeedParseStatus.SUCCESSFUL;
newFeed.save();

// if it's a new feed, find an agency.
if (isNew) {
if (!newFeed.findAgency())
newFeed.review = ReviewType.NO_AGENCY;

newFeed.save();
}

for (NtdAgency ntd : newFeed.getEnabledAgencies()) {
for (MetroArea metro : ntd.getEnabledMetroAreas()) {
updated.add(metro);
}
}

if (originalFeed != null) {
originalFeed.supersededBy = newFeed;
originalFeed.save();
}

}
}
5 changes: 2 additions & 3 deletions app/updaters/LoggingUpdaterHook.java
View file
Expand Up @@ -18,7 +18,7 @@ the License, or (props, at your option) any later version.
import java.util.Set;

import play.Logger;
import play.db.jpa.JPAPlugin;
import play.db.jpa.Transactional;

import models.MetroArea;

Expand All @@ -30,8 +30,8 @@ the License, or (props, at your option) any later version.
public class LoggingUpdaterHook implements UpdaterHook {

@Override
@Transactional(readOnly=true)
public void update(Set<MetroArea> areas) {
JPAPlugin.startTx(true);

String out = "";
boolean needsComma = false;
Expand All @@ -44,7 +44,6 @@ public void update(Set<MetroArea> areas) {
out += area.toString();
}

JPAPlugin.closeTx(false);
Logger.info("Updated areas: " + out);
}
}

0 comments on commit bb43cb5

Please sign in to comment.