feat: Date validity - Calendar Expiration + Trip Coverage

RULES.md

@@ -101,6 +101,7 @@ Each Notice is associated with a severity: `INFO`, `WARNING`, `ERROR`.
 | [`duplicate_route_name`](#duplicate_route_name)                                               | Two distinct routes have either the same `route_short_name`, the same `route_long_name`, or the same combination of `route_short_name` and `route_long_name`. |
 | [`empty_row`](#empty_row)                                                                     | A row in the input file has only spaces.                                                                                                                      |
 | [`equal_shape_distance_same_coordinates`](#equal_shape_distance_same_coordinates)             | Two consecutive points have equal `shape_dist_traveled` and the same lat/lon coordinates in `shapes.txt`.                                                     |
+| [`expired_calendar`](#expired_calendar)                                         | Dataset should not contain date ranges for services that have already expired.
 | [`fast_travel_between_consecutive_stops`](#fast_travel_between_consecutive_stops)             | A transit vehicle moves too fast between two consecutive stops.                                                                                               |
 | [`fast_travel_between_far_stops`](#fast_travel_between_far_stops)                             | A transit vehicle moves too fast between two far stops.                                                                                                       |
 | [`feed_expiration_date7_days`](#feed_expiration_date7_days)                                   | Dataset should be valid for at least the next 7 days.                                                                                                         |
@@ -134,6 +135,7 @@ Each Notice is associated with a severity: `INFO`, `WARNING`, `ERROR`.
 | [`stop_without_stop_time`](#stop_without_stop_time)                                           | A stop in `stops.txt` is not referenced by any `stop_times.stop_id`.                                                                                          |
 | [`transfer_with_suspicious_mid_trip_in_seat`](#transfer_with_suspicious_mid_trip_in_seat)     | A trip id field from GTFS file `transfers.txt` with an in-seat transfer type references a stop that is not in the expected position in the trip's stop-times. |
 | [`translation_unknown_table_name`](#translation_unknown_table_name)                           | A translation references an unknown or missing GTFS table.                                                                                                    |
+| [`trip_data_should_be_valid_for_next7_days`](#trip_data_should_be_valid_for_next7_days)   | Trips data should be valid for at least the next seven days.                                                                                            |
 | [`unexpected_enum_value`](#unexpected_enum_value)                                             | An enum has an unexpected value.                                                                                                                              |
 | [`unusable_trip`](#unusable_trip)                                                             | Trips must have more than one stop to be usable.                                                                                                              |
 | [`unused_shape`](#unused_shape)                                                               | Shape is not used in GTFS file `trips.txt`.                                                                                                                   |
@@ -1801,6 +1803,29 @@ When sorted by `shape.shape_pt_sequence`, the values for `shape_dist_traveled` m
 
 </details>
 
+<a name="ExpiredCalendarNotice"/>
+
+### expired_calendar
+
+ Dataset should not contain date ranges for services that have already expired. This warning takes into account the `calendar_dates.txt` file as well as the `calendar.txt` file.
+
+#### References
+* [Dataset Publishing & General Practices](https://gtfs.org/schedule/best-practices/#dataset-publishing-general-practices)
+
+<details>
+
+#### Notice fields description
+| Field name   	| Description                          	| Type    	|
+|--------------	|--------------------------------------	|---------	|
+| `csvRowNumber`| The row number of the faulty record. 	| Long 	    |
+| `serviceId`| The `service_id` for the faulty record. 	| Long 	    |
+
+#### Affected files
+[`calendar.txt`](https://gtfs.org/schedule/reference/#calendartxt)
+[`calendar_dates.txt`](https://gtfs.org/schedule/reference/#calendar_datestxt)
+
+</details>
+
 <a name="FastTravelBetweenConsecutiveStopsNotice"/>
 
 ### fast_travel_between_consecutive_stops
@@ -2653,6 +2678,32 @@ A translation references an unknown or missing GTFS table.
 * [`translations.txt`](http://gtfs.org/reference/static#translationstxt)
 
 </details>
+
+<a name="TripDataShouldBeValidForNext7DaysNotice"/>
+
+### trip_data_should_be_valid_for_next7_days
+
+Trips data should be valid for at least the next seven days.
+
+#### References
+
+- [Dataset Publishing & General Practices](https://gtfs.org/schedule/best-practices/#dataset-publishing-general-practices)
+<details>
+
+#### Notice fields description
+
+| Field name               | Description                                 | Type |
+| ------------------------ | ------------------------------------------- | ---- |
+| `currentDate`            | The date that the dataset was validated.    | Date |
+| `serviceWindowStartDate` | The start date of the trips in the dataset. | Date |
+| `serviceWindowEndDate`   | The end date of the trips in the dataset.   | Date |
+
+#### Affected files
+
+- [`trips.txt`](http://gtfs.org/reference/static#tripstxt)
+
+</details>
+
 <a name="UnexpectedEnumValueNotice"/>
 
 ### unexpected_enum_value

main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTripSchema.java

@@ -32,6 +32,7 @@ public interface GtfsTripSchema extends GtfsEntity {
   String routeId();
 
   @FieldType(FieldTypeEnum.ID)
+  @Index
   @Required
   String serviceId();
 

main/src/main/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidator.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2021 Jarvus Innovations LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.mobilitydata.gtfsvalidator.validator;
+
+import java.time.LocalDate;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Optional;
+import java.util.SortedSet;
+import javax.inject.Inject;
+import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator;
+import org.mobilitydata.gtfsvalidator.input.CurrentDateTime;
+import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
+import org.mobilitydata.gtfsvalidator.notice.SeverityLevel;
+import org.mobilitydata.gtfsvalidator.notice.ValidationNotice;
+import org.mobilitydata.gtfsvalidator.table.*;
+import org.mobilitydata.gtfsvalidator.type.GtfsDate;
+import org.mobilitydata.gtfsvalidator.util.CalendarUtil;
+import org.mobilitydata.gtfsvalidator.util.TripCalendarUtil;
+
+/**
+ * Validates that trip data exists for the next 7 days. Dates with the majority trips per day should
+ * be included for at least the next 7 day period. see
+ * https://github.com/MobilityData/gtfs-validator/issues/886#issuecomment-832237225
+ *
+ * <p>Generated notices:
+ *
+ * <ul>
+ *   <li>{@link TripDataShouldBeValidForNext7DaysNotice}.
+ * </ul>
+ */
+@GtfsValidator
+public class DateTripsValidator extends FileValidator {
+
+  private final GtfsCalendarDateTableContainer calendarDateTable;
+  private final GtfsCalendarTableContainer calendarTable;
+  private final GtfsTripTableContainer tripContainer;
+  private final GtfsFrequencyTableContainer frequencyTable;
+  private final CurrentDateTime currentDateTime;
+
+  @Inject
+  DateTripsValidator(
+      CurrentDateTime currentDateTime,
+      GtfsCalendarDateTableContainer calendarDateTable,
+      GtfsCalendarTableContainer calendarTable,
+      GtfsTripTableContainer tripContainer,
+      GtfsFrequencyTableContainer frequencyTable) {
+    this.currentDateTime = currentDateTime;
+    this.calendarTable = calendarTable;
+    this.calendarDateTable = calendarDateTable;
+    this.tripContainer = tripContainer;
+    this.frequencyTable = frequencyTable;
+  }
+
+  @Override
+  public void validate(NoticeContainer noticeContainer) {
+
+    LocalDate now = currentDateTime.getNow().toLocalDate();
+
+    final Map<String, SortedSet<LocalDate>> servicePeriodMap =
+        CalendarUtil.servicePeriodToServiceDatesMap(
+            CalendarUtil.buildServicePeriodMap(calendarTable, calendarDateTable));
+
+    NavigableMap<LocalDate, Integer> tripCounts =
+        TripCalendarUtil.countTripsForEachServiceDate(
+            servicePeriodMap, tripContainer, frequencyTable);
+    Optional<TripCalendarUtil.DateInterval> majorityServiceDates =
+        TripCalendarUtil.computeMajorityServiceCoverage(tripCounts);
+    LocalDate currentDatePlusSevenDays = now.plusDays(7);
+
+    if (!majorityServiceDates.isEmpty()) {
+      LocalDate serviceWindowStartDate = majorityServiceDates.get().startDate();
+      LocalDate serviceWindowEndDate = majorityServiceDates.get().endDate();
+      if (serviceWindowStartDate.isAfter(now)
+          || serviceWindowEndDate.isBefore(currentDatePlusSevenDays)) {
+        noticeContainer.addValidationNotice(
+            new TripDataShouldBeValidForNext7DaysNotice(
+                GtfsDate.fromLocalDate(now),
+                GtfsDate.fromLocalDate(serviceWindowStartDate),
+                GtfsDate.fromLocalDate(serviceWindowEndDate)));
+      }
+    }
+  }
+
+  static class TripDataShouldBeValidForNext7DaysNotice extends ValidationNotice {
+    // Current date (YYYYMMDD format)
+    private final GtfsDate currentDate;
+    // The start date of the majority service window.
+    private final GtfsDate serviceWindowStartDate;
+    // The end date of the majority service window.
+    private final GtfsDate serviceWindowEndDate;
+
+    TripDataShouldBeValidForNext7DaysNotice(
+        GtfsDate currentDate, GtfsDate serviceWindowStartDate, GtfsDate serviceWindowEndDate) {
+      super(SeverityLevel.WARNING);
+      this.currentDate = currentDate;
+      this.serviceWindowStartDate = serviceWindowStartDate;
+      this.serviceWindowEndDate = serviceWindowEndDate;
+    }
+  }
+}

main/src/main/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidator.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2021 Jarvus Innovations LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.mobilitydata.gtfsvalidator.validator;
+
+import java.time.LocalDate;
+import java.util.Map;
+import java.util.SortedSet;
+import javax.inject.Inject;
+import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator;
+import org.mobilitydata.gtfsvalidator.input.CurrentDateTime;
+import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
+import org.mobilitydata.gtfsvalidator.notice.SeverityLevel;
+import org.mobilitydata.gtfsvalidator.notice.ValidationNotice;
+import org.mobilitydata.gtfsvalidator.table.*;
+import org.mobilitydata.gtfsvalidator.util.CalendarUtil;
+
+@GtfsValidator
+public class ExpiredCalendarValidator extends FileValidator {
+  private final GtfsCalendarTableContainer calendarTable;
+  private final GtfsCalendarDateTableContainer calendarDateTable;
+  private final CurrentDateTime currentDateTime;
+
+  @Inject
+  ExpiredCalendarValidator(
+      CurrentDateTime currentDateTime,
+      GtfsCalendarTableContainer calendarTable,
+      GtfsCalendarDateTableContainer calendarDateTable) {
+    this.currentDateTime = currentDateTime;
+    this.calendarTable = calendarTable;
+    this.calendarDateTable = calendarDateTable;
+  }
+
+  @Override
+  public void validate(NoticeContainer noticeContainer) {
+    LocalDate now = currentDateTime.getNow().toLocalDate();
+
+    final Map<String, SortedSet<LocalDate>> servicePeriodMap =
+        CalendarUtil.servicePeriodToServiceDatesMap(
+            CalendarUtil.buildServicePeriodMap(calendarTable, calendarDateTable));
+
+    for (var serviceId : servicePeriodMap.keySet()) {
+      SortedSet<LocalDate> serviceDates = servicePeriodMap.get(serviceId);
+      LocalDate lastServiceDate = serviceDates.last();
+      if (lastServiceDate.isBefore(now)) {
+        int csvRowNumber = calendarTable.byServiceId(serviceId).get().csvRowNumber();
+        noticeContainer.addValidationNotice(new ExpiredCalendarNotice(csvRowNumber, serviceId));
+      }
+    }
+  }
+
+  static class ExpiredCalendarNotice extends ValidationNotice {
+    // The row of the faulty record.
+    private final int csvRowNumber;
+    // The service id of the faulty record.
+    private final String serviceId;
+
+    ExpiredCalendarNotice(int csvRowNumber, String serviceId) {
+      super(SeverityLevel.WARNING);
+      this.csvRowNumber = csvRowNumber;
+      this.serviceId = serviceId;
+    }
+  }
+}

main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java

@@ -0,0 +1,117 @@
+package org.mobilitydata.gtfsvalidator.validator;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mobilitydata.gtfsvalidator.validator.DateTripsValidator.*;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import java.time.DayOfWeek;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mobilitydata.gtfsvalidator.input.CurrentDateTime;
+import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
+import org.mobilitydata.gtfsvalidator.notice.ValidationNotice;
+import org.mobilitydata.gtfsvalidator.table.*;
+import org.mobilitydata.gtfsvalidator.type.GtfsDate;
+import org.mobilitydata.gtfsvalidator.util.CalendarUtilTest;
+
+@RunWith(JUnit4.class)
+public class DateTripsValidatorTest {
+  private static final ZonedDateTime TEST_NOW =
+      ZonedDateTime.of(2022, 12, 1, 8, 30, 0, 0, ZoneOffset.UTC);
+
+  @Test
+  public void serviceWindowEndingBefore7DaysFromNowShouldGenerateNotice() {
+
+    var serviceWindowStart = TEST_NOW.toLocalDate();
+    var serviceWindowEnd = TEST_NOW.toLocalDate().plusDays(6);
+
+    var notices = validateSimpleServiceWindow(serviceWindowStart, serviceWindowEnd);
+    assertThat(notices)
+        .containsExactly(
+            new TripDataShouldBeValidForNext7DaysNotice(
+                GtfsDate.fromLocalDate(TEST_NOW.toLocalDate()),
+                GtfsDate.fromLocalDate(serviceWindowStart),
+                GtfsDate.fromLocalDate(serviceWindowEnd)));
+  }
+
+  @Test
+  public void serviceWindowStartingAfterNowShouldGenerateNotice() {
+
+    var serviceWindowStart = TEST_NOW.toLocalDate().plusDays(1);
+    var serviceWindowEnd = TEST_NOW.toLocalDate().plusDays(7);
+    var notices = validateSimpleServiceWindow(serviceWindowStart, serviceWindowEnd);
+    assertThat(notices)
+        .containsExactly(
+            new TripDataShouldBeValidForNext7DaysNotice(
+                GtfsDate.fromLocalDate(TEST_NOW.toLocalDate()),
+                GtfsDate.fromLocalDate(serviceWindowStart),
+                GtfsDate.fromLocalDate(serviceWindowEnd)));
+  }
+
+  @Test
+  public void serviceWindowStartingNowAndEndingIn7DaysShouldNotGenerateNotice() {
+
+    var serviceWindowStart = TEST_NOW.toLocalDate();
+    var serviceWindowEnd = TEST_NOW.toLocalDate().plusDays(7);
+    var notices = validateSimpleServiceWindow(serviceWindowStart, serviceWindowEnd);
+    assertThat(notices).isEmpty();
+  }
+
+  @Test
+  public void serviceWindowStartingBeforeNowAndEndingAfter7DaysShouldNotGenerateNotice() {
+
+    var serviceWindowStart = TEST_NOW.toLocalDate().minusDays(1);
+    var serviceWindowEnd = TEST_NOW.toLocalDate().plusDays(8);
+    var notices = validateSimpleServiceWindow(serviceWindowStart, serviceWindowEnd);
+    assertThat(notices).isEmpty();
+  }
+
+  private List<ValidationNotice> validateSimpleServiceWindow(
+      LocalDate serviceWindowStart, LocalDate serviceWindowEnd) {
+    var serviceId = "s1";
+    var noticeContainer = new NoticeContainer();
+    var calendar =
+        CalendarUtilTest.createGtfsCalendar(
+            serviceId,
+            serviceWindowStart,
+            serviceWindowEnd,
+            ImmutableSet.copyOf(DayOfWeek.values()));
+    var calendarTable =
+        GtfsCalendarTableContainer.forEntities(ImmutableList.of(calendar), noticeContainer);
+    var dateTable = new GtfsCalendarDateTableContainer(GtfsTableContainer.TableStatus.EMPTY_FILE);
+    var frequencyTable = new GtfsFrequencyTableContainer(GtfsTableContainer.TableStatus.EMPTY_FILE);
+
+    var tripBlock = createTripBlock(serviceId, 6, "b1");
+    var tripContainer = GtfsTripTableContainer.forEntities(tripBlock, noticeContainer);
+
+    var validator =
+        new DateTripsValidator(
+            new CurrentDateTime(TEST_NOW), dateTable, calendarTable, tripContainer, frequencyTable);
+
+    validator.validate(noticeContainer);
+
+    return noticeContainer.getValidationNotices();
+  }
+
+  private static ArrayList<GtfsTrip> createTripBlock(
+      String serviceId, int tripsPerDay, String blockId) {
+    ArrayList<GtfsTrip> trips = new ArrayList<>();
+    for (int i = 0; i < tripsPerDay; i++) {
+      trips.add(
+          new GtfsTrip.Builder()
+              .setCsvRowNumber(i + 1)
+              .setTripId("t" + i)
+              .setServiceId(serviceId)
+              .setBlockId(blockId)
+              .build());
+    }
+    return trips;
+  }
+}