feat(MTCValidator): Implement MTC validator and complete test.

src/main/java/com/conveyal/gtfs/GTFS.java

@@ -16,10 +16,10 @@
 import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
 import org.apache.commons.dbcp2.PoolableConnectionFactory;
 import org.apache.commons.dbcp2.PoolingDataSource;
-import org.apache.commons.dbutils.DbUtils;
 import org.apache.commons.pool2.impl.GenericObjectPool;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import javax.sql.DataSource;
 import java.io.File;
 import java.io.IOException;
@@ -29,8 +29,8 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
 
 import static com.conveyal.gtfs.util.Util.ensureValidNamespace;
 
@@ -96,9 +96,13 @@ public static SnapshotResult makeSnapshot (String feedId, DataSource dataSource)
     /**
      * Once a feed has been loaded into the database, examine its contents looking for various problems and errors.
      */
-    public static ValidationResult validate (String feedId, DataSource dataSource, Class<? extends FeedValidator>... additionalValidators) {
+    public static ValidationResult validate (String feedId, DataSource dataSource) {
+        return validate(feedId, dataSource, new ArrayList<>());
+    }
+
+    public static ValidationResult validate (String feedId, DataSource dataSource, List<Class<? extends FeedValidator>> additionalValidatorClasses) {
         Feed feed = new Feed(dataSource, feedId);
-        ValidationResult result = feed.validate(Arrays.asList(additionalValidators));
+        ValidationResult result = feed.validate(additionalValidatorClasses);
         return result;
     }
 

src/main/java/com/conveyal/gtfs/validator/MTCValidator.java

@@ -2,7 +2,16 @@
 
 import com.conveyal.gtfs.error.SQLErrorStorage;
 import com.conveyal.gtfs.loader.Feed;
+import com.conveyal.gtfs.model.*;
 
+import static com.conveyal.gtfs.error.NewGTFSErrorType.FIELD_VALUE_TOO_LONG;
+
+/**
+ * MTCValidator validates a GTFS feed according to the following
+ * additional guidelines by 511 MTC:
+ * - Field values should not exceed some number of characters.
+ * Those requirements can be found by searching for the word 'character'.
+ */
 public class MTCValidator extends FeedValidator {
 
     public MTCValidator(Feed feed, SQLErrorStorage errorStorage) {
@@ -11,6 +20,30 @@ public MTCValidator(Feed feed, SQLErrorStorage errorStorage) {
 
     @Override
     public void validate() {
+        for (Agency agency : feed.agencies) {
+            fieldLengthShouldNotExceed(agency, agency.agency_id, 50);
+            fieldLengthShouldNotExceed(agency, agency.agency_name, 50);
+            fieldLengthShouldNotExceed(agency, agency.agency_url, 500);
+        }
+
+        for (Stop stop : feed.stops) {
+            fieldLengthShouldNotExceed(stop, stop.stop_name, 100);
+        }
+
+        for (Trip trip : feed.trips) {
+            fieldLengthShouldNotExceed(trip, trip.trip_headsign, 120);
+            fieldLengthShouldNotExceed(trip, trip.trip_short_name, 50);
+        }
+
+        // TODO: Handle calendar_attributes.txt?
+    }
 
+    boolean fieldLengthShouldNotExceed(Entity entity, Object objValue, int maxLength) {
+        String value = objValue != null ? objValue.toString() : "";
+        if (value.length() > maxLength) {
+            if (errorStorage != null) registerError(entity, FIELD_VALUE_TOO_LONG, value);
+            return false;
+        }
+        return true;
     }
 }

src/test/java/com/conveyal/gtfs/GTFSTest.java

@@ -9,12 +9,13 @@
 import com.conveyal.gtfs.storage.PersistenceExpectation;
 import com.conveyal.gtfs.storage.RecordExpectation;
 import com.conveyal.gtfs.util.InvalidNamespaceException;
+import com.conveyal.gtfs.validator.FeedValidator;
+import com.conveyal.gtfs.validator.MTCValidator;
 import com.conveyal.gtfs.validator.ValidationResult;
 import com.csvreader.CsvReader;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Multimap;
 import com.google.common.io.Files;
-import org.apache.commons.dbutils.DbUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.input.BOMInputStream;
 import org.hamcrest.Matcher;
@@ -32,14 +33,11 @@
 import java.io.PrintStream;
 import java.nio.charset.Charset;
 import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.zip.ZipEntry;
@@ -59,6 +57,8 @@ public class GTFSTest {
     private static final String JDBC_URL = "jdbc:postgresql://localhost";
     private static final Logger LOG = LoggerFactory.getLogger(GTFSTest.class);
 
+    private List<Class<? extends FeedValidator>> extraValidatorClasses = new ArrayList<>();
+
     // setup a stream to capture the output from the program
     @Before
     public void setUpStreams() {
@@ -306,6 +306,30 @@ public void canLoadAndExportSimpleAgencyWithOnlyCalendarDates() {
         );
     }
 
+    /**
+     * Tests that a GTFS feed with long field values generates corresponding
+     * validation errors per MTC guidelines.
+     */
+    @Test
+    public void canLoadFeedWithLongFieldValues () {
+        extraValidatorClasses.add(MTCValidator.class);
+
+        PersistenceExpectation[] expectations = PersistenceExpectation.list();
+        ErrorExpectation[] errorExpectations = ErrorExpectation.list(
+                new ErrorExpectation(NewGTFSErrorType.FIELD_VALUE_TOO_LONG),
+                new ErrorExpectation(NewGTFSErrorType.FIELD_VALUE_TOO_LONG),
+                new ErrorExpectation(NewGTFSErrorType.FIELD_VALUE_TOO_LONG),
+                new ErrorExpectation(NewGTFSErrorType.FIELD_VALUE_TOO_LONG),
+                new ErrorExpectation(NewGTFSErrorType.FIELD_VALUE_TOO_LONG),
+                new ErrorExpectation(NewGTFSErrorType.FIELD_VALUE_TOO_LONG),
+                new ErrorExpectation(NewGTFSErrorType.FEED_TRAVEL_TIMES_ROUNDED) // Not related, not worrying about this one.
+        );
+        assertThat(
+                "Long-field-value test passes",
+                runIntegrationTestOnFolder("fake-agency-mtc-long-fields", nullValue(), expectations, errorExpectations),
+                equalTo(true)
+        );
+    }
 
     /**
      * A helper method that will zip a specified folder in test/main/resources and call
@@ -359,7 +383,7 @@ private boolean runIntegrationTestOnZipFile(
             // load and validate feed
             LOG.info("load and validate GTFS file {}", zipFileName);
             FeedLoadResult loadResult = GTFS.load(zipFileName, dataSource);
-            ValidationResult validationResult = GTFS.validate(loadResult.uniqueIdentifier, dataSource);
+            ValidationResult validationResult = GTFS.validate(loadResult.uniqueIdentifier, dataSource, extraValidatorClasses);
 
             assertThat(validationResult.fatalException, is(fatalExceptionExpectation));
             namespace = loadResult.uniqueIdentifier;