diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index daa3a663a..071259596 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -32,11 +32,11 @@ jobs: with: java-version: 19 distribution: 'temurin' - # Install node 20 for running e2e tests (and for maven-semantic-release). - - name: Use Node.js 20.x + # Install node 22 for running e2e tests (and for maven-semantic-release). + - name: Use Node.js 22.x uses: actions/setup-node@v1 with: - node-version: 20.x + node-version: 22.x - name: Start MongoDB uses: supercharge/mongodb-github-action@1.3.0 with: @@ -98,10 +98,10 @@ jobs: # Run maven-semantic-release to potentially create a new release of datatools-server. The flag --skip-maven-deploy is # used to avoid deploying to maven central. So essentially, this just creates a release with a changelog on github. - - name: Use Node.js 20.x + - name: Use Node.js 22.x uses: actions/setup-node@v1 with: - node-version: 20.x + node-version: 22.x - name: Run maven-semantic-release if: env.SAVE_JAR_TO_S3 == 'true' env: diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java index 28f4fec28..6336813c2 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java @@ -4,6 +4,7 @@ import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; +import com.conveyal.datatools.manager.jobs.ValidateGtfsPlusFeedJob; import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; @@ -365,6 +366,7 @@ public static boolean validateAll (boolean load, boolean force, String filterFee } else { JobUtils.heavyExecutor.execute(new ValidateFeedJob(version, systemUser, false)); JobUtils.heavyExecutor.execute(new ValidateMobilityDataFeedJob(version, systemUser, false)); + JobUtils.heavyExecutor.execute(new ValidateGtfsPlusFeedJob(version, systemUser, false)); } } // ValidateAllFeedsJob validateAllFeedsJob = new ValidateAllFeedsJob("system", force, load); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java index d04556dc2..1b783088a 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java @@ -242,13 +242,22 @@ private static String publishGtfsPlusFile(Request req, Response res) { } /** - * HTTP endpoint that validates GTFS+ tables for a specific feed version (or its saved/edited GTFS+). + * HTTP endpoint that validates GTFS+ tables for a specific feed version (or its saved/edited GTFS+). If the feed + * version already has GTFS+ validation results, those will be returned instead of re-validating. */ private static GtfsPlusValidation getGtfsPlusValidation(Request req, Response res) { String feedVersionId = req.params("versionid"); GtfsPlusValidation gtfsPlusValidation = null; try { + FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); + if (feedVersion != null && feedVersion.gtfsPlusValidation != null) { + return feedVersion.gtfsPlusValidation; + } gtfsPlusValidation = GtfsPlusValidation.validate(feedVersionId); + if (feedVersion != null) { + feedVersion.gtfsPlusValidation = gtfsPlusValidation; + Persistence.feedVersions.replace(feedVersion.id, feedVersion); + } } catch(Exception e) { logMessageAndHalt(req, 500, "Could not read GTFS+ zip file", e); } diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index 0de656b91..33f50476e 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -38,7 +38,7 @@ public class GtfsPlusValidation implements Serializable { private static final String REALTIME_ROUTES_TXT = "realtime_routes.txt"; // Public fields to appear in validation JSON. - public final String feedVersionId; + public String feedVersionId; /** Indicates whether GTFS+ validation applies to user-edited feed or original published GTFS feed */ public boolean published; public long lastModified; @@ -49,22 +49,36 @@ private GtfsPlusValidation (String feedVersionId) { this.feedVersionId = feedVersionId; } + public GtfsPlusValidation(boolean published, List issues) { + this.published = published; + this.issues = issues; + } + + public GtfsPlusValidation() { + // Empty constructor for serialization + } + /** - * Validate a GTFS+ feed and return a list of issues encountered. - * FIXME: For now this uses the MapDB-backed GTFSFeed class. Which actually suggests that this might - * should be contained within a MonitorableJob. + * Overload method to retrieve the feed version. */ public static GtfsPlusValidation validate(String feedVersionId) throws Exception { - GtfsPlusValidation validation = new GtfsPlusValidation(feedVersionId); + FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); + return validate(feedVersion); + } + + /** + * Validate a GTFS+ feed and return a list of issues encountered. + */ + public static GtfsPlusValidation validate(FeedVersion feedVersion) throws Exception { if (!DataManager.isModuleEnabled("gtfsplus")) { throw new IllegalStateException("GTFS+ module must be enabled in server.yml to run GTFS+ validation."); } - LOG.info("Validating GTFS+ for {}", feedVersionId); + GtfsPlusValidation validation = new GtfsPlusValidation(feedVersion.id); + LOG.info("Validating GTFS+ for {}", feedVersion.id); - FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId); // Load the main GTFS file. // FIXME: Swap MapDB-backed GTFSFeed for use of SQL data? - File gtfsFeedDbFile = gtfsPlusStore.getFeedFile(feedVersionId + ".db"); + File gtfsFeedDbFile = gtfsPlusStore.getFeedFile(feedVersion.id + ".db"); String gtfsFeedDbFilePath = gtfsFeedDbFile.getAbsolutePath(); GTFSFeed gtfsFeed; try { @@ -87,7 +101,7 @@ public static GtfsPlusValidation validate(String feedVersionId) throws Exception } // check for saved GTFS+ data - File file = gtfsPlusStore.getFeed(feedVersionId); + File file = gtfsPlusStore.getFeed(feedVersion.id); if (file == null) { validation.published = true; LOG.warn("GTFS+ Validation -- Modified GTFS+ file not found, loading from main version GTFS."); diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java index b835a3996..34c59b578 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java @@ -10,6 +10,10 @@ public class ValidationIssue implements Serializable { public int rowIndex; public String description; + public ValidationIssue() { + // Empty constructor for serialization + } + public ValidationIssue(String tableId, String fieldName, int rowIndex, String description) { this.tableId = tableId; this.fieldName = fieldName; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 52f28f8ba..d73b62851 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -41,7 +41,7 @@ public class ProcessSingleFeedJob extends FeedVersionJob { public static boolean ENABLE_MTC_TRANSFORMATIONS = true; // Used in testing to skip validation and speed up response times. - public static boolean VALIDATE_MOBILITY_DATA = true; + public static boolean ENABLE_ADDITIONAL_VALIDATION = true; /** * Create a job for the given feed version. @@ -131,8 +131,9 @@ public void jobLogic() { // Next, validate the feed. addNextJob(new ValidateFeedJob(feedVersion, owner, isNewVersion)); - if (VALIDATE_MOBILITY_DATA) { + if (ENABLE_ADDITIONAL_VALIDATION) { addNextJob(new ValidateMobilityDataFeedJob(feedVersion, owner, isNewVersion)); + addNextJob(new ValidateGtfsPlusFeedJob(feedVersion, owner, isNewVersion)); } // We only need to snapshot the feed if there are transformations at the database level. In the case that there diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java new file mode 100644 index 000000000..4255f3ae0 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java @@ -0,0 +1,70 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.common.status.FeedVersionJob; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This job handles the Gtfs+ validation of a given feed version. If the version is not new, it will simply + * replace the existing version with the version object that has updated validation info. + */ +public class ValidateGtfsPlusFeedJob extends FeedVersionJob { + public static final Logger LOG = LoggerFactory.getLogger(ValidateGtfsPlusFeedJob.class); + + private final FeedVersion feedVersion; + private final boolean isNewVersion; + + public ValidateGtfsPlusFeedJob(FeedVersion version, Auth0UserProfile owner, boolean isNewVersion) { + super(owner, "Validating Gtfs+", JobType.VALIDATE_FEED); + feedVersion = version; + this.isNewVersion = isNewVersion; + status.update("Waiting to begin Gtfs+ validation...", 0); + } + + @Override + public void jobLogic () { + LOG.info("Running ValidateGtfsPlusFeedJob for {}", feedVersion.id); + feedVersion.validateGtfsPlus(status); + } + + @Override + public void jobFinished () { + if (!status.error) { + if (parentJobId != null && JobType.PROCESS_FEED.equals(parentJobType)) { + // Validate stage is happening as part of an overall process feed job. + // At this point all GTFS data has been loaded and validated, so we record + // the FeedVersion into mongo. + // This happens here because otherwise we would have to wait for other jobs, + // such as BuildTransportNetwork, to finish. If those subsequent jobs fail, + // the version won't get loaded into MongoDB (even though it exists in postgres). + feedVersion.persistFeedVersionAfterValidation(isNewVersion); + } + status.completeSuccessfully("Gtfs+ validation finished!"); + } else { + // If the version was not stored successfully, call FeedVersion#delete to reset things to before the version + // was uploaded/fetched. Note: delete calls made to MongoDB on the version ID will not succeed, but that is + // expected. + feedVersion.delete(); + } + } + + /** + * Getter that allows a client to know the ID of the feed version that will be created as soon as the upload is + * initiated; however, we will not store the FeedVersion in the mongo application database until the upload and + * processing is completed. This prevents clients from manipulating GTFS data before it is entirely imported. + */ + @JsonProperty + public String getFeedVersionId () { + return feedVersion.id; + } + + @JsonProperty + public String getFeedSourceId () { + return feedVersion.parentFeedSource().id; + } + + +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java index 1b88b91a5..0ec3abab7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSourceSummary.java @@ -1,8 +1,12 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.validator.ValidationResult; + +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.collect.Lists; @@ -32,8 +36,8 @@ import static com.mongodb.client.model.Filters.in; public class FeedSourceSummary { + private static final ObjectMapper mapper = new ObjectMapper(); private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); - public String projectId; public String id; @@ -71,6 +75,18 @@ public class FeedSourceSummary { public String organizationId; + public Date latestProcessedByExternalPublisher; + + public Date latestSentToExternalPublisher; + + public GtfsPlusValidation latestGtfsPlusValidation; + + public String latestPublishedVersionId; + + public String latestNamespace; + + public FeedValidationResultSummary publishedValidationSummary; + public FeedSourceSummary() { } @@ -111,6 +127,15 @@ public void setFeedVersion(FeedVersionSummary feedVersionSummary, boolean isDepl : feedVersionSummary.validationResult.errorCount; } else { this.latestValidation = new LatestValidationResult(feedVersionSummary); + this.latestProcessedByExternalPublisher = feedVersionSummary.processedByExternalPublisher; + this.latestSentToExternalPublisher = feedVersionSummary.sentToExternalPublisher; + this.latestGtfsPlusValidation = feedVersionSummary.gtfsPlusValidation; + this.latestNamespace = feedVersionSummary.namespace; + this.latestPublishedVersionId = feedVersionSummary.publishedVersionId; + this.publishedValidationSummary = new FeedValidationResultSummary(); + this.publishedValidationSummary.errorCount = feedVersionSummary.publishedFeedVersionErrorCount; + this.publishedValidationSummary.startDate = feedVersionSummary.publishedFeedVersionStartDate; + this.publishedValidationSummary.endDate = feedVersionSummary.publishedFeedVersionEndDate; } } } @@ -197,18 +222,35 @@ public static Map getLatestFeedVersionForFeedSources } }, { - $unwind: "$feedVersions" + $lookup: { + from: "FeedVersion", + localField: "publishedVersionId", + foreignField: "namespace", + as: "publishedFeedVersion" + } + }, + { + $unwind: "$feedVersions", + $unwind: "$publishedFeedVersion", }, { $group: { _id: "$_id", - doc: { + publishedVersionId: { $first: "$publishedVersionId" }, + publishedFeedVersionErrorCount: { $first: "$publishedFeedVersion.validationResult.errorCount"}, + publishedFeedVersionStartDate: { $first: "$publishedFeedVersion.validationResult.firstCalendarDate"}, + publishedFeedVersionEndDate: { $first: "$publishedFeedVersion.validationResult.lastCalendarDate"}, + feedVersion: { $max: { version: "$feedVersions.version", feedVersionId: "$feedVersions._id", firstCalendarDate: "$feedVersions.validationResult.firstCalendarDate", lastCalendarDate: "$feedVersions.validationResult.lastCalendarDate", - issues: "$feedVersions.validationResult.errorCount" + errorCount: "$feedVersions.validationResult.errorCount", + processedByExternalPublisher: "$feedVersions.processedByExternalPublisher", + sentToExternalPublisher: "$feedVersions.sentToExternalPublisher", + gtfsPlusValidation: "$feedVersions.gtfsPlusValidation", + namespace: "$feedVersions.namespace" } } } @@ -220,13 +262,23 @@ public static Map getLatestFeedVersionForFeedSources in("projectId", projectId) ), lookup("FeedVersion", "_id", "feedSourceId", "feedVersions"), + lookup("FeedVersion", "publishedVersionId", "namespace", "publishedFeedVersion"), unwind("$feedVersions"), + unwind("$publishedFeedVersion"), group( "$_id", + Accumulators.first("publishedVersionId", "$publishedVersionId"), + Accumulators.first("publishedFeedVersionErrorCount", "$publishedFeedVersion.validationResult.errorCount"), + Accumulators.first("publishedFeedVersionStartDate", "$publishedFeedVersion.validationResult.firstCalendarDate"), + Accumulators.first("publishedFeedVersionEndDate", "$publishedFeedVersion.validationResult.lastCalendarDate"), Accumulators.last("feedVersionId", "$feedVersions._id"), Accumulators.last("firstCalendarDate", "$feedVersions.validationResult.firstCalendarDate"), Accumulators.last("lastCalendarDate", "$feedVersions.validationResult.lastCalendarDate"), - Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount") + Accumulators.last("errorCount", "$feedVersions.validationResult.errorCount"), + Accumulators.last("processedByExternalPublisher", "$feedVersions.processedByExternalPublisher"), + Accumulators.last("sentToExternalPublisher", "$feedVersions.sentToExternalPublisher"), + Accumulators.last("gtfsPlusValidation", "$feedVersions.gtfsPlusValidation"), + Accumulators.last("namespace", "$feedVersions.namespace") ) ); return extractFeedVersionSummaries( @@ -461,15 +513,58 @@ private static Map extractFeedVersionSummaries( List stages ) { Map feedVersionSummaries = new HashMap<>(); - for (Document feedVersionDocument : Persistence.getDocuments(collection, stages)) { + Document feedVersionDocument = Persistence.getDocuments(collection, stages).first(); + if (feedVersionDocument != null) { + // Only expected one or zero documents to be returned. FeedVersionSummary feedVersionSummary = new FeedVersionSummary(); feedVersionSummary.id = feedVersionDocument.getString(feedVersionKey); + feedVersionSummary.processedByExternalPublisher = feedVersionDocument.getDate("processedByExternalPublisher"); + feedVersionSummary.sentToExternalPublisher = feedVersionDocument.getDate("sentToExternalPublisher"); + feedVersionSummary.gtfsPlusValidation = getGtfsPlusValidation(feedVersionDocument); + feedVersionSummary.namespace = feedVersionDocument.getString("namespace"); feedVersionSummary.validationResult = getValidationResult(hasChildValidationResultDocument, feedVersionDocument); + + // The feed source's published version id. + feedVersionSummary.publishedVersionId = feedVersionDocument.getString("publishedVersionId"); + + // The feed source's published feed version. Feed source's publishedVersionId mapped to feed version's namespace. + feedVersionSummary.publishedFeedVersionErrorCount = feedVersionDocument.get("publishedFeedVersionErrorCount") != null + ? feedVersionDocument.getInteger("publishedFeedVersionErrorCount") + : 0; + feedVersionSummary.publishedFeedVersionStartDate = feedVersionDocument.get("publishedFeedVersionStartDate") != null + ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionStartDate")) + : null; + feedVersionSummary.publishedFeedVersionEndDate = feedVersionDocument.get("publishedFeedVersionEndDate") != null + ? getDateFromString(feedVersionDocument.getString("publishedFeedVersionEndDate")) + : null; feedVersionSummaries.put(feedVersionDocument.getString(feedSourceKey), feedVersionSummary); } return feedVersionSummaries; } + /** + * Build GtfsPlusValidation object from feed version document. + */ + private static GtfsPlusValidation getGtfsPlusValidation(Document feedVersionDocument) { + Document gtfsPlusValidationDocument = getDocumentChild(feedVersionDocument, "gtfsPlusValidation"); + if (gtfsPlusValidationDocument == null) return null; + List issues = null; + if (gtfsPlusValidationDocument.containsKey("issues") && gtfsPlusValidationDocument.get("issues") != null) { + List issueDocs = gtfsPlusValidationDocument.getList("issues", Document.class); + if (issueDocs != null) { + issues = new ArrayList<>(); + for (Document doc : issueDocs) { + issues.add(mapper.convertValue(doc, ValidationIssue.class)); + } + } + } + boolean published = false; + if (gtfsPlusValidationDocument.getBoolean("published") != null) { + published = gtfsPlusValidationDocument.getBoolean("published"); + } + return new GtfsPlusValidation(published, issues); + } + /** * Build validation result from feed version document. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 8452d63d7..93095f204 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -5,6 +5,7 @@ import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.extensions.mtc.MtcFeedResource; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.jobs.ValidateFeedJob; import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob; import com.conveyal.datatools.manager.jobs.validation.RouteTypeValidatorBuilder; @@ -276,6 +277,8 @@ public FeedValidationResultSummary validationSummary() { public Document mobilityDataResult; + public GtfsPlusValidation gtfsPlusValidation; + public String formattedTimestamp() { SimpleDateFormat format = new SimpleDateFormat(HUMAN_READABLE_TIMESTAMP_FORMAT); return format.format(this.updated); @@ -477,6 +480,28 @@ public void validateMobility(MonitorableJob.Status status) { } } + /** + * Produce Gtfs+ validation results for this feed version if GTFS+ module is enabled. + */ + public void validateGtfsPlus(MonitorableJob.Status status) { + + // Sometimes this method is called when no status object is available. + if (status == null) status = new MonitorableJob.Status(); + + if (DataManager.isModuleEnabled("gtfsplus")) { + try { + gtfsPlusValidation = GtfsPlusValidation.validate(this); + } catch (Exception e) { + LOG.warn("Unable to validate GTFS+ validation.", e); + status.fail(String.format("Unable to validate feed %s", this.id), e); + validationResult = new ValidationResult(); + validationResult.fatalException = "failure!"; + } + } else { + LOG.warn("GTFS+ module not enabled, skipping GTFS+ validation."); + } + } + public void validate() { validate(null); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java index be2413a9a..f9879a3d1 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersionSummary.java @@ -1,6 +1,7 @@ package com.conveyal.datatools.manager.models; import com.conveyal.datatools.editor.utils.JacksonSerializers; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -28,6 +29,13 @@ public class FeedVersionSummary extends Model implements Serializable { @JsonIgnore public ValidationResult validationResult; private PartialValidationSummary validationSummary; + public Date processedByExternalPublisher; + public Date sentToExternalPublisher; + public GtfsPlusValidation gtfsPlusValidation; + public String publishedVersionId; + public int publishedFeedVersionErrorCount; + public LocalDate publishedFeedVersionStartDate; + public LocalDate publishedFeedVersionEndDate; public PartialValidationSummary getValidationSummary() { if (validationSummary == null) { diff --git a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java index 9648d0982..994536d53 100644 --- a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java +++ b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java @@ -47,7 +47,7 @@ static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = false; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; project = new Project(); project.name = appendDate("Test"); Persistence.projects.create(project); @@ -86,7 +86,7 @@ static void setUp() throws IOException { @AfterAll static void tearDown() throws SQLException, InvalidNamespaceException, CheckedAWSException { Auth0Connection.setAuthDisabled(false); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = true; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; project.delete(); FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionOrphan.id); if (feedVersion != null) { diff --git a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java index 1350fbe52..7e37debfd 100644 --- a/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java +++ b/src/test/java/com/conveyal/datatools/manager/controllers/api/FeedSourceControllerTest.java @@ -4,6 +4,8 @@ import com.conveyal.datatools.TestUtils; import com.conveyal.datatools.common.utils.Scheduler; import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.datatools.manager.gtfsplus.ValidationIssue; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; @@ -55,6 +57,7 @@ public class FeedSourceControllerTest extends DatatoolsTest { private static Project projectWithLatestDeployment = null; private static FeedSource feedSourceWithLatestDeploymentFeedVersion = null; private static FeedVersion feedVersionFromLatestDeployment = null; + private static FeedVersion feedVersionPublishedFromLatestDeployment = null; private static Deployment deploymentLatest = null; private static Deployment deploymentSuperseded = null; @@ -67,6 +70,7 @@ public class FeedSourceControllerTest extends DatatoolsTest { public static void setUp() throws IOException { DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); + project = new Project(); project.name = "ProjectOne"; project.autoFetchFeeds = true; @@ -121,7 +125,16 @@ private static void setUpFeedVersionFromLatestDeployment() throws MalformedURLEx "feed-version-from-latest-deployment", feedSourceWithLatestDeploymentFeedVersion.id, deployedStartDate, - deployedEndDate + deployedEndDate, + null + ); + feedVersionPublishedFromLatestDeployment = createFeedVersion( + "published-feed-version-from-latest-deployment", + // Set to null so the relationship to feed source is via the published version id. + null, + LocalDate.of(2022, Month.NOVEMBER, 2), + LocalDate.of(2022, Month.NOVEMBER, 3), + feedSourceWithLatestDeploymentFeedVersion.publishedVersionId ); deploymentSuperseded = createDeployment( "deployment-superseded", @@ -218,6 +231,9 @@ private static void tearDownDeployedFeedVersion() { if (feedVersionFromLatestDeployment != null) { Persistence.feedVersions.removeById(feedVersionFromLatestDeployment.id); } + if (feedVersionPublishedFromLatestDeployment != null) { + Persistence.feedVersions.removeById(feedVersionPublishedFromLatestDeployment.id); + } if (feedVersionFromPinnedDeployment != null) { Persistence.feedVersions.removeById(feedVersionFromPinnedDeployment.id); } @@ -397,6 +413,17 @@ void canRetrieveDeployedFeedVersionFromLatestDeployment() throws IOException { assertEquals(feedVersionFromLatestDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); assertEquals(feedVersionFromLatestDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); + assertEquals(feedVersionFromLatestDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); + assertEquals(feedVersionFromLatestDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); + assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); + assertEquals(feedVersionFromLatestDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); + assertEquals(feedVersionFromLatestDeployment.namespace, feedSourceSummaries.get(0).latestNamespace); + + assertEquals(feedSourceWithPinnedDeploymentFeedVersion.publishedVersionId, feedSourceSummaries.get(0).latestPublishedVersionId); + + assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.errorCount, feedSourceSummaries.get(0).publishedValidationSummary.errorCount); + assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.firstCalendarDate, feedSourceSummaries.get(0).publishedValidationSummary.startDate); + assertEquals(feedVersionPublishedFromLatestDeployment.validationResult.lastCalendarDate, feedSourceSummaries.get(0).publishedValidationSummary.endDate); } @Test @@ -431,6 +458,10 @@ void canRetrieveDeployedFeedVersionFromPinnedDeployment() throws IOException { assertEquals(feedVersionFromPinnedDeployment.validationSummary().startDate, feedSourceSummaries.get(0).latestValidation.startDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().endDate, feedSourceSummaries.get(0).latestValidation.endDate); assertEquals(feedVersionFromPinnedDeployment.validationSummary().errorCount, feedSourceSummaries.get(0).latestValidation.errorCount); + assertEquals(feedVersionFromPinnedDeployment.processedByExternalPublisher, feedSourceSummaries.get(0).latestProcessedByExternalPublisher); + assertEquals(feedVersionFromPinnedDeployment.sentToExternalPublisher, feedSourceSummaries.get(0).latestSentToExternalPublisher); + assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.issues.size(), feedSourceSummaries.get(0).latestGtfsPlusValidation.issues.size()); + assertEquals(feedVersionFromPinnedDeployment.gtfsPlusValidation.published, feedSourceSummaries.get(0).latestGtfsPlusValidation.published); } private static FeedSource createFeedSource(String name, URL url, Project project) { @@ -461,6 +492,7 @@ private static FeedSource createFeedSource( feedSource.projectId = project.id; feedSource.retrievalMethod = FeedRetrievalMethod.FETCHED_AUTOMATICALLY; feedSource.url = url; + feedSource.publishedVersionId = "published-version-id-1"; if (labels != null) feedSource.labelIds = labels; if (notes != null) feedSource.noteIds = notes; if (persist) Persistence.feedSources.create(feedSource); @@ -489,13 +521,17 @@ private static Deployment createDeployment( * Helper method to create a feed version with no start date. */ private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate endDate) { - return createFeedVersion(id, feedSourceId, null, endDate); + return createFeedVersion(id, feedSourceId, null, endDate, null); + } + + private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate endDate, String namespace) { + return createFeedVersion(id, feedSourceId, null, endDate, namespace); } /** * Helper method to create a feed version. */ - private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate startDate, LocalDate endDate) { + private static FeedVersion createFeedVersion(String id, String feedSourceId, LocalDate startDate, LocalDate endDate, String namespace) { FeedVersion feedVersion = new FeedVersion(); feedVersion.id = id; feedVersion.feedSourceId = feedSourceId; @@ -504,6 +540,14 @@ private static FeedVersion createFeedVersion(String id, String feedSourceId, Loc validationResult.lastCalendarDate = endDate; validationResult.errorCount = 5 + (int)(Math.random() * ((1000 - 5) + 1)); feedVersion.validationResult = validationResult; + feedVersion.processedByExternalPublisher = new Date(); + feedVersion.sentToExternalPublisher = new Date(); + List issues = List.of( + new ValidationIssue("Test issue 1", "stops.txt", 1, "stop_id"), + new ValidationIssue("Test issue 2", "stops.txt", 2, "stop_id") + ); + feedVersion.gtfsPlusValidation = new GtfsPlusValidation(true, issues); + feedVersion.namespace = namespace != null ? namespace : "feed-version-namespace"; Persistence.feedVersions.create(feedVersion); return feedVersion; } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 7b383075e..39340905d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -87,7 +87,7 @@ public class MergeFeedsJobTest extends UnitTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = false; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = false; // Enable MTC extension, but disable the transformations. ProcessSingleFeedJob.ENABLE_MTC_TRANSFORMATIONS = false; DatatoolsTest.enableMTCExtension(); @@ -169,7 +169,7 @@ public static void tearDown() { } DatatoolsTest.resetMTCExtension(); ProcessSingleFeedJob.ENABLE_MTC_TRANSFORMATIONS = true; - ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = true; + ProcessSingleFeedJob.ENABLE_ADDITIONAL_VALIDATION = true; } /**