From 3827fd20608db20b13f62dfe269dae7079eef318 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 31 May 2023 15:19:02 -0400 Subject: [PATCH 1/7] Added missing_recommended_column notice and removed missing_timepoint_column notice. --- RULES.md | 259 +++++++++--------- .../MissingRecommendedColumnNotice.java | 39 +++ .../gtfsvalidator/notice/Notice.java | 12 + .../gtfsvalidator/notice/NoticeContainer.java | 11 + .../gtfsvalidator/table/AnyTableLoader.java | 4 + .../table/GtfsColumnDescriptor.java | 4 + .../DefaultTableHeaderValidator.java | 20 +- .../validator/TableHeaderValidator.java | 1 + .../table/AnyTableLoaderTest.java | 3 + .../testgtfs/GtfsTestTableDescriptor.java | 2 + .../validator/TableHeaderValidatorTest.java | 26 ++ .../table/GtfsStopTimeSchema.java | 2 + .../validator/TimepointTimeValidator.java | 22 +- .../validator/TimepointTimeValidatorTest.java | 17 +- .../annotation/RecommendedColumn.java | 50 ++++ .../gtfsvalidator/processor/Analyser.java | 1 + .../processor/GtfsFieldDescriptor.java | 4 + .../processor/TableDescriptorGenerator.java | 2 + .../RecommendedColumnAnnotationSchema.java | 13 + 19 files changed, 339 insertions(+), 153 deletions(-) create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/notice/MissingRecommendedColumnNotice.java create mode 100644 model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/RecommendedColumn.java create mode 100644 processor/tests/src/main/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchema.java diff --git a/RULES.md b/RULES.md index 3ef9211289..f335a482a0 100644 --- a/RULES.md +++ b/RULES.md @@ -26,119 +26,119 @@ Each Notice is associated with a severity: `INFO`, `WARNING`, `ERROR`. ## Table of ERRORS -| Notice code | Description | -|-----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`block_trips_with_overlapping_stop_times`](#block_trips_with_overlapping_stop_times) | Block trips with overlapping stop times. | -| [`csv_parsing_failed`](#csv_parsing_failed) | Parsing of a CSV file failed. | -| [`decreasing_shape_distance`](#decreasing_shape_distance) | Decreasing `shape_dist_traveled` in `shapes.txt`. | -| [`decreasing_or_equal_stop_time_distance`](#decreasing_or_equal_stop_time_distance) | Decreasing or equal `shape_dist_traveled` in `stop_times.txt`. | -| [`duplicated_column`](#duplicated_column) | Duplicated column in CSV. | -| [`duplicate_key`](#duplicate_key) | Duplicated entity. | -| [`empty_column_name`](#empty_column_name) | A column name is empty. | -| [`empty_file`](#empty_file) | A CSV file is empty. | -| [`equal_shape_distance_diff_coordinates`](#equal_shape_distance_diff_coordinates) | Two consecutive points have equal `shape_dist_traveled` and different lat/lon coordinates in `shapes.txt`. | -| [`fare_transfer_rule_duration_limit_type_without_duration_limit`](#fare_transfer_rule_duration_limit_type_without_duration_limit) | A row from GTFS file `fare_transfer_rules.txt` has a defined `duration_limit_type` field but no `duration_limit` specified. | -| [`fare_transfer_rule_duration_limit_without_type`](#fare_transfer_rule_duration_limit_without_type) | A row from GTFS file `fare_transfer_rules.txt` has a defined `duration_limit` field but no `duration_limit_type` specified. | -| [`fare_transfer_rule_invalid_transfer_count`](#fare_transfer_rule_invalid_transfer_count) | A row from GTFS file `fare_transfer_rules.txt` has a defined `transfer_count` with an invalid value. | -| [`fare_transfer_rule_missing_transfer_count`](#fare_transfer_rule_missing_transfer_count) | A row from `fare_transfer_rules.txt` has `from_leg_group_id` equal to `to_leg_group_id`, but has no `transfer_count` specified. | -| [`fare_transfer_rule_with_forbidden_transfer_count`](#fare_transfer_rule_with_forbidden_transfer_count) | A row from `fare_transfer_rules.txt` has `from_leg_group_id` not equal to `to_leg_group_id`, but has `transfer_count` specified. | -| [`foreign_key_violation`](#foreign_key_violation) | Wrong foreign key. | -| [`inconsistent_agency_timezone`](#inconsistent_agency_timezone) | Inconsistent Timezone among agencies. | -| [`invalid_color`](#invalid_color) | A field contains an invalid color value. | -| [`invalid_currency`](#invalid_currency) | A field contains a wrong currency code. | -| [`invalid_currency_amount`](#invalid_currency_amount) | A currency amount field has a value that does not match the format of its corresponding currency code field. | -| [`invalid_date`](#invalid_date) | A field cannot be parsed as date. | -| [`invalid_email`](#invalid_email) | A field contains a malformed email address. | -| [`invalid_float`](#invalid_float) | A field cannot be parsed as a floating point number. | -| [`invalid_integer`](#invalid_integer) | A field cannot be parsed as an integer. | -| [`invalid_language_code`](#invalid_language_code) | A field contains a wrong language code. | -| [`invalid_phone_number`](#invalid_phone_number) | A field contains a malformed phone number. | -| [`invalid_row_length`](#invalid_row_length) | Invalid csv row length. | -| [`invalid_time`](#invalid_time) | A field cannot be parsed as time. | -| [`invalid_timezone`](#invalid_timezone) | A field cannot be parsed as a timezone. | -| [`invalid_url`](#invalid_url) | A field contains a malformed URL. | -| [`location_without_parent_station`](#location_without_parent_station) | A location that must have `parent_station` field does not have it. | -| [`location_with_unexpected_stop_time`](#location_with_unexpected_stop_time) | A location in `stops.txt` that is not a stop is referenced by some `stop_times.stop_id`. | -| [`missing_calendar_and_calendar_date_files`](#missing_calendar_and_calendar_date_files) | Missing GTFS files `calendar.txt` and `calendar_dates.txt`. | -| [`missing_level_id`](#missing_level_id) | `stops.level_id` is conditionally required. | -| [`missing_required_column`](#missing_required_column) | A required column is missing in the input file. | -| [`missing_required_field`](#missing_required_field) | A required field is missing. | -| [`missing_required_file`](#missing_required_file) | A required file is missing. | -| [`missing_stop_name`](#missing_stop_name) | `stops.stop_name` is required for `location_type` equal to `0`, `1`, or `2`. | -| [`missing_trip_edge`](#missing_trip_edge) | Missing trip edge `arrival_time` or `departure_time`. | -| [`new_line_in_value`](#new_line_in_value) | New line or carriage return in a value in CSV file. | -| [`number_out_of_range`](#number_out_of_range) | Out of range value. | -| [`overlapping_frequency`](#overlapping_frequency) | Trip frequencies overlap. | -| [`pathway_to_platform_with_boarding_areas`](#pathway_to_platform_with_boarding_areas) | A pathway has an endpoint that is a platform which has boarding areas. | -| [`pathway_to_wrong_location_type`](#pathway_to_wrong_location_type) | A pathway has an endpoint that is a station. | -| [`pathway_unreachable_location`](#pathway_unreachable_location) | A location is not reachable at least in one direction: from the entrances or to the exits. | -| [`point_near_origin`](#point_near_origin) | A point is too close to origin `(0, 0)`. | -| [`point_near_pole`](#point_near_pole) | A point is too close to the North or South Pole. | -| [`route_both_short_and_long_name_missing`](#route_both_short_and_long_name_missing) | Missing route short name and long name. | -| [`start_and_end_range_equal`](#start_and_end_range_equal) | Two date or time fields are equal. | -| [`start_and_end_range_out_of_order`](#start_and_end_range_out_of_order) | Two date or time fields are out of order. | -| [`station_with_parent_station`](#station_with_parent_station) | A station has `parent_station` field set. | -| [`stop_time_timepoint_without_times`](#stop_time_timepoint_without_times) | `arrival_time` or `departure_time` not specified for timepoint. | -| [`stop_time_with_arrival_before_previous_departure_time`](#stop_time_with_arrival_before_previous_departure_time) | Backwards time travel between stops in `stop_times.txt` | -| [`stop_time_with_only_arrival_or_departure_time`](#stop_time_with_only_arrival_or_departure_time) | Missing `stop_times.arrival_time` or `stop_times.departure_time`. | -| [`stop_without_location`](#stop_without_location) | `stop_lat` and/or `stop_lon` is missing for stop with `location_type` equal to`0`, `1`, or `2` | -| [`stop_without_zone_id`](#stop_without_zone_id) | Stop without value for `stops.zone_id`. | -| [`too_many_rows`](#too_many_rows) | A CSV file has too many rows. | -| [`transfer_with_invalid_stop_location_type`](#transfer_with_invalid_stop_location_type) | A stop id field from GTFS file `transfers.txt` references a stop that has a `location_type` other than 0 or 1 (aka Stop/Platform or Station). | -| [`transfer_with_invalid_trip_and_route`](#transfer_with_invalid_trip_and_route) | A trip id field from GTFS file `transfers.txt` references a route that does not match its `trips.txt` `route_id`. | -| [`transfer_with_invalid_trip_and_stop`](#transfer_with_invalid_trip_and_stop) | A trip id field from GTFS file `transfers.txt` references a stop that is not included in the referenced trip's stop-times. | -| [`translation_foreign_key_violation`](#translation_foreign_key_violation) | An entity with the given `record_id` and `record_sub_id` cannot be found in the referenced table. | -| [`translation_unexpected_value`](#translation_unexpected_value) | A field in a translations row has value but must be empty. | -| [`wrong_parent_location_type`](#wrong_parent_location_type) | Incorrect type of the parent location. | +| Notice code | Description | +|-----------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| [`block_trips_with_overlapping_stop_times`](#block_trips_with_overlapping_stop_times) | Block trips with overlapping stop times. | +| [`csv_parsing_failed`](#csv_parsing_failed) | Parsing of a CSV file failed. | +| [`decreasing_shape_distance`](#decreasing_shape_distance) | Decreasing `shape_dist_traveled` in `shapes.txt`. | +| [`decreasing_or_equal_stop_time_distance`](#decreasing_or_equal_stop_time_distance) | Decreasing or equal `shape_dist_traveled` in `stop_times.txt`. | +| [`duplicated_column`](#duplicated_column) | Duplicated column in CSV. | +| [`duplicate_key`](#duplicate_key) | Duplicated entity. | +| [`empty_column_name`](#empty_column_name) | A column name is empty. | +| [`empty_file`](#empty_file) | A CSV file is empty. | +| [`equal_shape_distance_diff_coordinates`](#equal_shape_distance_diff_coordinates) | Two consecutive points have equal `shape_dist_traveled` and different lat/lon coordinates in `shapes.txt`. | +| [`fare_transfer_rule_duration_limit_type_without_duration_limit`](#fare_transfer_rule_duration_limit_type_without_duration_limit) | A row from GTFS file `fare_transfer_rules.txt` has a defined `duration_limit_type` field but no `duration_limit` specified. | +| [`fare_transfer_rule_duration_limit_without_type`](#fare_transfer_rule_duration_limit_without_type) | A row from GTFS file `fare_transfer_rules.txt` has a defined `duration_limit` field but no `duration_limit_type` specified. | +| [`fare_transfer_rule_invalid_transfer_count`](#fare_transfer_rule_invalid_transfer_count) | A row from GTFS file `fare_transfer_rules.txt` has a defined `transfer_count` with an invalid value. | +| [`fare_transfer_rule_missing_transfer_count`](#fare_transfer_rule_missing_transfer_count) | A row from `fare_transfer_rules.txt` has `from_leg_group_id` equal to `to_leg_group_id`, but has no `transfer_count` specified. | +| [`fare_transfer_rule_with_forbidden_transfer_count`](#fare_transfer_rule_with_forbidden_transfer_count) | A row from `fare_transfer_rules.txt` has `from_leg_group_id` not equal to `to_leg_group_id`, but has `transfer_count` specified. | +| [`foreign_key_violation`](#foreign_key_violation) | Wrong foreign key. | +| [`inconsistent_agency_timezone`](#inconsistent_agency_timezone) | Inconsistent Timezone among agencies. | +| [`invalid_color`](#invalid_color) | A field contains an invalid color value. | +| [`invalid_currency`](#invalid_currency) | A field contains a wrong currency code. | +| [`invalid_currency_amount`](#invalid_currency_amount) | A currency amount field has a value that does not match the format of its corresponding currency code field. | +| [`invalid_date`](#invalid_date) | A field cannot be parsed as date. | +| [`invalid_email`](#invalid_email) | A field contains a malformed email address. | +| [`invalid_float`](#invalid_float) | A field cannot be parsed as a floating point number. | +| [`invalid_integer`](#invalid_integer) | A field cannot be parsed as an integer. | +| [`invalid_language_code`](#invalid_language_code) | A field contains a wrong language code. | +| [`invalid_phone_number`](#invalid_phone_number) | A field contains a malformed phone number. | +| [`invalid_row_length`](#invalid_row_length) | Invalid csv row length. | +| [`invalid_time`](#invalid_time) | A field cannot be parsed as time. | +| [`invalid_timezone`](#invalid_timezone) | A field cannot be parsed as a timezone. | +| [`invalid_url`](#invalid_url) | A field contains a malformed URL. | +| [`location_without_parent_station`](#location_without_parent_station) | A location that must have `parent_station` field does not have it. | +| [`location_with_unexpected_stop_time`](#location_with_unexpected_stop_time) | A location in `stops.txt` that is not a stop is referenced by some `stop_times.stop_id`. | +| [`missing_calendar_and_calendar_date_files`](#missing_calendar_and_calendar_date_files) | Missing GTFS files `calendar.txt` and `calendar_dates.txt`. | +| [`missing_level_id`](#missing_level_id) | `stops.level_id` is conditionally required. | +| [`missing_required_column`](#missing_required_column) | A required column is missing in the input file. | +| [`missing_required_field`](#missing_required_field) | A required field is missing. | +| [`missing_required_file`](#missing_required_file) | A required file is missing. | +| [`missing_stop_name`](#missing_stop_name) | `stops.stop_name` is required for `location_type` equal to `0`, `1`, or `2`. | +| [`missing_trip_edge`](#missing_trip_edge) | Missing trip edge `arrival_time` or `departure_time`. | +| [`new_line_in_value`](#new_line_in_value) | New line or carriage return in a value in CSV file. | +| [`number_out_of_range`](#number_out_of_range) | Out of range value. | +| [`overlapping_frequency`](#overlapping_frequency) | Trip frequencies overlap. | +| [`pathway_to_platform_with_boarding_areas`](#pathway_to_platform_with_boarding_areas) | A pathway has an endpoint that is a platform which has boarding areas. | +| [`pathway_to_wrong_location_type`](#pathway_to_wrong_location_type) | A pathway has an endpoint that is a station. | +| [`pathway_unreachable_location`](#pathway_unreachable_location) | A location is not reachable at least in one direction: from the entrances or to the exits. | +| [`point_near_origin`](#point_near_origin) | A point is too close to origin `(0, 0)`. | +| [`point_near_pole`](#point_near_pole) | A point is too close to the North or South Pole. | +| [`route_both_short_and_long_name_missing`](#route_both_short_and_long_name_missing) | Missing route short name and long name. | +| [`start_and_end_range_equal`](#start_and_end_range_equal) | Two date or time fields are equal. | +| [`start_and_end_range_out_of_order`](#start_and_end_range_out_of_order) | Two date or time fields are out of order. | +| [`station_with_parent_station`](#station_with_parent_station) | A station has `parent_station` field set. | +| [`stop_time_timepoint_without_times`](#stop_time_timepoint_without_times) | `arrival_time` or `departure_time` not specified for timepoint. | +| [`stop_time_with_arrival_before_previous_departure_time`](#stop_time_with_arrival_before_previous_departure_time) | Backwards time travel between stops in `stop_times.txt` | +| [`stop_time_with_only_arrival_or_departure_time`](#stop_time_with_only_arrival_or_departure_time) | Missing `stop_times.arrival_time` or `stop_times.departure_time`. | +| [`stop_without_location`](#stop_without_location) | `stop_lat` and/or `stop_lon` is missing for stop with `location_type` equal to`0`, `1`, or `2` | +| [`stop_without_zone_id`](#stop_without_zone_id) | Stop without value for `stops.zone_id`. | +| [`too_many_rows`](#too_many_rows) | A CSV file has too many rows. | +| [`transfer_with_invalid_stop_location_type`](#transfer_with_invalid_stop_location_type) | A stop id field from GTFS file `transfers.txt` references a stop that has a `location_type` other than 0 or 1 (aka Stop/Platform or Station). | +| [`transfer_with_invalid_trip_and_route`](#transfer_with_invalid_trip_and_route) | A trip id field from GTFS file `transfers.txt` references a route that does not match its `trips.txt` `route_id`. | +| [`transfer_with_invalid_trip_and_stop`](#transfer_with_invalid_trip_and_stop) | A trip id field from GTFS file `transfers.txt` references a stop that is not included in the referenced trip's stop-times. | +| [`translation_foreign_key_violation`](#translation_foreign_key_violation) | An entity with the given `record_id` and `record_sub_id` cannot be found in the referenced table. | +| [`translation_unexpected_value`](#translation_unexpected_value) | A field in a translations row has value but must be empty. | +| [`wrong_parent_location_type`](#wrong_parent_location_type) | Incorrect type of the parent location. | ## Table of WARNINGS -| Notice code | Description | -|-----------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`attribution_without_role`](#attribution_without_role) | Attribution with no role. | -| [`duplicate_fare_media`](#duplicate_fare_media) | Two distinct fare media have the same fare media name and type. | -| [`duplicate_route_name`](#duplicate_route_name) | Two distinct routes have either the same `route_short_name`, the same `route_long_name`, or the same combination of `route_short_name` and `route_long_name`. | -| [`empty_row`](#empty_row) | A row in the input file has only spaces. | -| [`equal_shape_distance_same_coordinates`](#equal_shape_distance_same_coordinates) | Two consecutive points have equal `shape_dist_traveled` and the same lat/lon coordinates in `shapes.txt`. | -| [`expired_calendar`](#expired_calendar) | Dataset should not contain date ranges for services that have already expired. | -| [`fast_travel_between_consecutive_stops`](#fast_travel_between_consecutive_stops) | A transit vehicle moves too fast between two consecutive stops. | -| [`fast_travel_between_far_stops`](#fast_travel_between_far_stops) | A transit vehicle moves too fast between two far stops. | -| [`feed_expiration_date7_days`](#feed_expiration_date7_days) | Dataset should be valid for at least the next 7 days. | -| [`feed_expiration_date30_days`](#feed_expiration_date30_days) | Dataset should cover at least the next 30 days of service. | -| [`feed_info_lang_and_agency_lang_mismatch`](#feed_info_lang_and_agency_lang_mismatch) | Mismatching feed and agency language fields. | -| [`inconsistent_agency_lang`](#inconsistent_agency_lang) | Inconsistent language among agencies. | -| [`leading_or_trailing_whitespaces`](#leading_or_trailing_whitespaces) | The value in CSV file has leading or trailing whitespaces. | -| [`missing_feed_info_date`](#missing_feed_info_date) | `feed_end_date` should be provided if `feed_start_date` is provided. `feed_start_date` should be provided if `feed_end_date` is provided. | -| [`missing_recommended_file`](#missing_recommended_file) | A recommended file is missing. | -| [`missing_recommended_field`](#missing_recommended_field) | A recommended field is missing. | -| [`missing_timepoint_column`](#missing_timepoint_column) | `timepoint` column is missing for a dataset. | -| [`missing_timepoint_value`](#missing_timepoint_value) | `stop_times.timepoint` value is missing for a record. | -| [`mixed_case_recommended_field`](#mixed_case_recommended_field) | This field has customer-facing text and should use Mixed Case (should contain upper and lower case letters). | -| [`more_than_one_entity`](#more_than_one_entity) | More than one row in CSV. | -| [`non_ascii_or_non_printable_char`](#non_ascii_or_non_printable_char) | Non ascii or non printable char in `id`. | -| [`pathway_dangling_generic_node`](#pathway_dangling_generic_node) | A generic node has only one incident location in a pathway graph. | -| [`pathway_loop`](#pathway_loop) | A pathway starts and ends at the same location. | -| [`route_color_contrast`](#route_color_contrast) | Insufficient route color contrast. | -| [`route_long_name_contains_short_name`](#route_long_name_contains_short_name) | Long name should not contain short name for a single route. | -| [`route_short_name_too_long`](#route_short_name_too_long) | Short name of a route is too long (more than 12 characters). | -| [`same_name_and_description_for_route`](#same_name_and_description_for_route) | Same name and description for route. | -| [`same_name_and_description_for_stop`](#same_name_and_description_for_stop) | Same name and description for stop. | -| [`same_route_and_agency_url`](#same_route_and_agency_url) | Same `routes.route_url` and `agency.agency_url`. | -| [`same_stop_and_agency_url`](#same_stop_and_agency_url) | Same `stops.stop_url` and `agency.agency_url`. | -| [`same_stop_and_route_url`](#same_stop_and_route_url) | Same `stops.stop_url` and `routes.route_url`. | -| [`stop_has_too_many_matches_for_shape`](#stop_has_too_many_matches_for_shape) | Stop entry that has many potential matches to the trip's path of travel. | -| [`stops_match_shape_out_of_order`](#stops_match_shape_out_of_order) | Two stop entries are different than their arrival-departure order defined by the shapes.txt | -| [`stop_too_far_from_shape`](#stop_too_far_from_shape) | Stop too far from trip shape. | -| [`stop_too_far_from_shape_using_user_distance`](#stop_too_far_from_shape_using_user_distance) | Stop time too far from shape. | -| [`stop_without_stop_time`](#stop_without_stop_time) | A stop in `stops.txt` is not referenced by any `stop_times.stop_id`. | -| [`transfer_with_suspicious_mid_trip_in_seat`](#transfer_with_suspicious_mid_trip_in_seat) | A trip id field from GTFS file `transfers.txt` with an in-seat transfer type references a stop that is not in the expected position in the trip's stop-times. | -| [`translation_unknown_table_name`](#translation_unknown_table_name) | A translation references an unknown or missing GTFS table. | -| [`trip_coverage_not_active_for_next7_days`](#trip_coverage_not_active_for_next7_days) | Trips data should be valid for at least the next seven days. | -| [`unexpected_enum_value`](#unexpected_enum_value) | An enum has an unexpected value. | -| [`unusable_trip`](#unusable_trip) | Trips must have more than one stop to be usable. | -| [`unused_shape`](#unused_shape) | Shape is not used in GTFS file `trips.txt`. | -| [`unused_trip`](#unused_trip) | Trip is not be used in `stop_times.txt` | +| Notice code | Description | +|------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`attribution_without_role`](#attribution_without_role) | Attribution with no role. | +| [`duplicate_fare_media`](#duplicate_fare_media) | Two distinct fare media have the same fare media name and type. | +| [`duplicate_route_name`](#duplicate_route_name) | Two distinct routes have either the same `route_short_name`, the same `route_long_name`, or the same combination of `route_short_name` and `route_long_name`. | +| [`empty_row`](#empty_row) | A row in the input file has only spaces. | +| [`equal_shape_distance_same_coordinates`](#equal_shape_distance_same_coordinates) | Two consecutive points have equal `shape_dist_traveled` and the same lat/lon coordinates in `shapes.txt`. | +| [`expired_calendar`](#expired_calendar) | Dataset should not contain date ranges for services that have already expired. | +| [`fast_travel_between_consecutive_stops`](#fast_travel_between_consecutive_stops) | A transit vehicle moves too fast between two consecutive stops. | +| [`fast_travel_between_far_stops`](#fast_travel_between_far_stops) | A transit vehicle moves too fast between two far stops. | +| [`feed_expiration_date7_days`](#feed_expiration_date7_days) | Dataset should be valid for at least the next 7 days. | +| [`feed_expiration_date30_days`](#feed_expiration_date30_days) | Dataset should cover at least the next 30 days of service. | +| [`feed_info_lang_and_agency_lang_mismatch`](#feed_info_lang_and_agency_lang_mismatch) | Mismatching feed and agency language fields. | +| [`inconsistent_agency_lang`](#inconsistent_agency_lang) | Inconsistent language among agencies. | +| [`leading_or_trailing_whitespaces`](#leading_or_trailing_whitespaces) | The value in CSV file has leading or trailing whitespaces. | +| [`missing_feed_info_date`](#missing_feed_info_date) | `feed_end_date` should be provided if `feed_start_date` is provided. `feed_start_date` should be provided if `feed_end_date` is provided. | +| [`missing_recommended_column`](#missing_recommended_column) | A recommended column is missing in the input file. | +| [`missing_recommended_file`](#missing_recommended_file) | A recommended file is missing. | +| [`missing_recommended_field`](#missing_recommended_field) | A recommended field is missing. | +| [`missing_timepoint_value`](#missing_timepoint_value) | `stop_times.timepoint` value is missing for a record. | +| [`mixed_case_recommended_field`](#mixed_case_recommended_field) | This field has customer-facing text and should use Mixed Case (should contain upper and lower case letters). | +| [`more_than_one_entity`](#more_than_one_entity) | More than one row in CSV. | +| [`non_ascii_or_non_printable_char`](#non_ascii_or_non_printable_char) | Non ascii or non printable char in `id`. | +| [`pathway_dangling_generic_node`](#pathway_dangling_generic_node) | A generic node has only one incident location in a pathway graph. | +| [`pathway_loop`](#pathway_loop) | A pathway starts and ends at the same location. | +| [`route_color_contrast`](#route_color_contrast) | Insufficient route color contrast. | +| [`route_long_name_contains_short_name`](#route_long_name_contains_short_name) | Long name should not contain short name for a single route. | +| [`route_short_name_too_long`](#route_short_name_too_long) | Short name of a route is too long (more than 12 characters). | +| [`same_name_and_description_for_route`](#same_name_and_description_for_route) | Same name and description for route. | +| [`same_name_and_description_for_stop`](#same_name_and_description_for_stop) | Same name and description for stop. | +| [`same_route_and_agency_url`](#same_route_and_agency_url) | Same `routes.route_url` and `agency.agency_url`. | +| [`same_stop_and_agency_url`](#same_stop_and_agency_url) | Same `stops.stop_url` and `agency.agency_url`. | +| [`same_stop_and_route_url`](#same_stop_and_route_url) | Same `stops.stop_url` and `routes.route_url`. | +| [`stop_has_too_many_matches_for_shape`](#stop_has_too_many_matches_for_shape) | Stop entry that has many potential matches to the trip's path of travel. | +| [`stops_match_shape_out_of_order`](#stops_match_shape_out_of_order) | Two stop entries are different than their arrival-departure order defined by the shapes.txt | +| [`stop_too_far_from_shape`](#stop_too_far_from_shape) | Stop too far from trip shape. | +| [`stop_too_far_from_shape_using_user_distance`](#stop_too_far_from_shape_using_user_distance) | Stop time too far from shape. | +| [`stop_without_stop_time`](#stop_without_stop_time) | A stop in `stops.txt` is not referenced by any `stop_times.stop_id`. | +| [`transfer_with_suspicious_mid_trip_in_seat`](#transfer_with_suspicious_mid_trip_in_seat) | A trip id field from GTFS file `transfers.txt` with an in-seat transfer type references a stop that is not in the expected position in the trip's stop-times. | +| [`translation_unknown_table_name`](#translation_unknown_table_name) | A translation references an unknown or missing GTFS table. | +| [`trip_coverage_not_active_for_next7_days`](#trip_coverage_not_active_for_next7_days) | Trips data should be valid for at least the next seven days. | +| [`unexpected_enum_value`](#unexpected_enum_value) | An enum has an unexpected value. | +| [`unusable_trip`](#unusable_trip) | Trips must have more than one stop to be usable. | +| [`unused_shape`](#unused_shape) | Shape is not used in GTFS file `trips.txt`. | +| [`unused_trip`](#unused_trip) | Trip is not be used in `stop_times.txt` | @@ -2092,6 +2092,28 @@ Even though `feed_info.start_date` and `feed_info.end_date` are optional, if one + + +### missing_recommended_column + +A recommended column is missing in the input file. + +#### References +* [GTFS terms definition](https://gtfs.org/reference/static/#term-definitions) +
+ +#### Notice fields description +| Field name | Description | Type | +|------------- |-------------------------------- |-------- | +| `filename` | The name of the faulty file. | String | +| `fieldName` | The name of the missing column. | String | + +#### Affected files +[All GTFS files supported by the specification.](http://gtfs.org/reference/static#dataset-files) + +
+ + ### missing_recommended_file @@ -2137,25 +2159,6 @@ The given field has no value in some input row, even though values are recommend - - -### missing_timepoint_column - -The `timepoint` column should be provided. - -#### References -* [stop_times.txt best practices](https://gtfs.org/schedule/best-practices/#stop_timestxt) -
- -#### Notice fields description -| Field name | Description | Type | -|---------------- |------------------------------------------------- |-------- | -| `filename` | The name of the affected file. | String | - -#### Affected files -* [`stop_times.txt`](https://gtfs.org/schedule/reference/#stop_timestxt) -
- ### missing_timepoint_value diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/MissingRecommendedColumnNotice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/MissingRecommendedColumnNotice.java new file mode 100644 index 0000000000..f40588daae --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/MissingRecommendedColumnNotice.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mobilitydata.gtfsvalidator.notice; + +import static org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef.TERM_DEFINITIONS; +import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.WARNING; + +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; +import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRefs; + +/** A recommended column is missing in the input file. */ +@GtfsValidationNotice(severity = WARNING, sections = @SectionRefs(TERM_DEFINITIONS)) +public class MissingRecommendedColumnNotice extends ValidationNotice { + + /** The name of the faulty file. */ + private final String filename; + + /** The name of the missing column. */ + private final String fieldName; + + public MissingRecommendedColumnNotice(String filename, String fieldName) { + super(SeverityLevel.WARNING); + this.filename = filename; + this.fieldName = fieldName; + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/Notice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/Notice.java index ab8db68cb6..43a50febb7 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/Notice.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/Notice.java @@ -127,6 +127,18 @@ public boolean isError() { return getSeverityLevel().ordinal() >= SeverityLevel.ERROR.ordinal(); } + /** + * Tells if this notice is a {@code WARNING}. + * + *

This method is preferred to checking {@code severityLevel} directly since more levels may be + * added in the future. + * + * @return true if this notice is an warning, false otherwise + */ + public boolean isWarning() { + return getSeverityLevel().ordinal() == SeverityLevel.WARNING.ordinal(); + } + /** JSON exclusion strategy for notice context. It skips {@link Notice#severityLevel}. */ private static class NoticeExclusionStrategy implements ExclusionStrategy { public boolean shouldSkipClass(Class clazz) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoticeContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoticeContainer.java index b0ad1d338b..35f66d22af 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoticeContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoticeContainer.java @@ -58,6 +58,7 @@ public class NoticeContainer { private final List systemErrors = new ArrayList<>(); private final Map noticesCountPerTypeAndSeverity = new HashMap<>(); private boolean hasValidationErrors = false; + private boolean hasValidationWarnings = false; /** * Used to specify limits on amount of notices in this {@code NoticeContainer}. @@ -91,6 +92,10 @@ public void addValidationNotice(ValidationNotice notice) { if (notice.isError()) { hasValidationErrors = true; } + if (notice.isWarning()) { + hasValidationWarnings = true; + } + updateNoticeCount(notice); if (validationNotices.size() >= maxTotalValidationNotices || noticesCountPerTypeAndSeverity.get(notice.getMappingKey()) @@ -138,6 +143,7 @@ public void addAll(NoticeContainer otherContainer) { validationNotices.addAll(otherContainer.validationNotices); systemErrors.addAll(otherContainer.systemErrors); hasValidationErrors |= otherContainer.hasValidationErrors; + hasValidationWarnings |= otherContainer.hasValidationWarnings; for (Entry entry : otherContainer.noticesCountPerTypeAndSeverity.entrySet()) { int count = noticesCountPerTypeAndSeverity.getOrDefault(entry.getKey(), 0); noticesCountPerTypeAndSeverity.put(entry.getKey(), count + entry.getValue()); @@ -149,6 +155,11 @@ public boolean hasValidationErrors() { return hasValidationErrors; } + /** Tells if this container has any {@code ValidationNotice} that is a warning. */ + public boolean hasValidationWarnings() { + return hasValidationWarnings; + } + /** Returns a list of all validation notices in the container. */ public List getValidationNotices() { return Collections.unmodifiableList(validationNotices); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java index 70a1da8012..567641330c 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java @@ -144,6 +144,10 @@ private static NoticeContainer validateHeaders( .filter(GtfsColumnDescriptor::headerRequired) .map(GtfsColumnDescriptor::columnName) .collect(Collectors.toSet()), + columnDescriptors.stream() + .filter(GtfsColumnDescriptor::headerRecommended) + .map(GtfsColumnDescriptor::columnName) + .collect(Collectors.toSet()), headerNotices); return headerNotices; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsColumnDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsColumnDescriptor.java index d75c87933d..0546e479d1 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsColumnDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsColumnDescriptor.java @@ -11,6 +11,8 @@ public abstract class GtfsColumnDescriptor { public abstract boolean headerRequired(); + public abstract boolean headerRecommended(); + public abstract FieldLevelEnum fieldLevel(); public abstract Optional numberBounds(); @@ -33,6 +35,8 @@ public abstract static class Builder { public abstract Builder setHeaderRequired(boolean value); + public abstract Builder setHeaderRecommended(boolean value); + public abstract Builder setFieldLevel(FieldLevelEnum value); public abstract Builder setNumberBounds(Optional value); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultTableHeaderValidator.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultTableHeaderValidator.java index 6f9b993270..3ef0f174c6 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultTableHeaderValidator.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultTableHeaderValidator.java @@ -23,6 +23,7 @@ import java.util.TreeSet; import org.mobilitydata.gtfsvalidator.notice.DuplicatedColumnNotice; import org.mobilitydata.gtfsvalidator.notice.EmptyColumnNameNotice; +import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedColumnNotice; import org.mobilitydata.gtfsvalidator.notice.MissingRequiredColumnNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.UnknownColumnNotice; @@ -36,14 +37,20 @@ public void validate( CsvHeader actualHeader, Set supportedColumns, Set requiredColumns, + Set recommendedColumns, NoticeContainer noticeContainer) { if (actualHeader.getColumnCount() == 0) { // This is an empty file. return; } Map columnIndices = new HashMap<>(); - // Sorted tree set for stable order of notices. + // Sorted tree set of all the columns for stable order of notices. + // We remove the columns that are properly present and well formed from that set, and at the + // end only the missing required columns are left in the set. TreeSet missingColumns = new TreeSet<>(requiredColumns); + // We also want to find the recommended columns that are absent. We use the same scheme for + // these. + TreeSet missingRecommendedColumns = new TreeSet<>(recommendedColumns); for (int i = 0; i < actualHeader.getColumnCount(); ++i) { String column = actualHeader.getColumnName(i); // Column indices are zero-based. We add 1 to make them 1-based. @@ -59,12 +66,23 @@ public void validate( if (!supportedColumns.contains(column)) { noticeContainer.addValidationNotice(new UnknownColumnNotice(filename, column, i + 1)); } + + // If the column is present, it should not be in the missing required columns set missingColumns.remove(column); + + // If the column is present, it should not be in the missing recommended columns set + missingRecommendedColumns.remove(column); } if (!missingColumns.isEmpty()) { for (String column : missingColumns) { noticeContainer.addValidationNotice(new MissingRequiredColumnNotice(filename, column)); } } + + if (!missingRecommendedColumns.isEmpty()) { + for (String column : missingRecommendedColumns) { + noticeContainer.addValidationNotice(new MissingRecommendedColumnNotice(filename, column)); + } + } } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidator.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidator.java index ef42d43770..3cc5732d2d 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidator.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidator.java @@ -28,5 +28,6 @@ void validate( CsvHeader actualHeader, Set supportedHeaders, Set requiredHeaders, + Set recommendedHeaders, NoticeContainer noticeContainer); } diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java index e804caaa40..07b9adc865 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java @@ -89,6 +89,7 @@ public void validate( CsvHeader actualHeader, Set supportedHeaders, Set requiredHeaders, + Set recommendedHeaders, NoticeContainer noticeContainer) { noticeContainer.addValidationNotice(headerValidationNotice); } @@ -145,6 +146,7 @@ public void missingRequiredField() { GtfsColumnDescriptor.builder() .setColumnName(GtfsTestEntity.ID_FIELD_NAME) .setHeaderRequired(true) + .setHeaderRecommended(false) .setFieldLevel(FieldLevelEnum.REQUIRED) .setIsMixedCase(false) .setIsCached(false) @@ -152,6 +154,7 @@ public void missingRequiredField() { GtfsColumnDescriptor.builder() .setColumnName(GtfsTestEntity.CODE_FIELD_NAME) .setHeaderRequired(false) + .setHeaderRecommended(false) .setFieldLevel(FieldLevelEnum.REQUIRED) .setIsMixedCase(false) .setIsCached(false) diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java index e202139dd9..76dbdad6b7 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java @@ -40,6 +40,7 @@ public ImmutableList getColumns() { GtfsColumnDescriptor.builder() .setColumnName(GtfsTestEntity.ID_FIELD_NAME) .setHeaderRequired(true) + .setHeaderRecommended(false) .setFieldLevel(FieldLevelEnum.REQUIRED) .setIsMixedCase(false) .setIsCached(false) @@ -48,6 +49,7 @@ public ImmutableList getColumns() { GtfsColumnDescriptor.builder() .setColumnName(GtfsTestEntity.CODE_FIELD_NAME) .setHeaderRequired(false) + .setHeaderRecommended(false) .setFieldLevel(FieldLevelEnum.OPTIONAL) .setIsMixedCase(false) .setIsCached(false) diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidatorTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidatorTest.java index a26323d187..4dfd474832 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidatorTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidatorTest.java @@ -19,11 +19,13 @@ import static com.google.common.truth.Truth.assertThat; import com.google.common.collect.ImmutableSet; +import java.util.Set; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.mobilitydata.gtfsvalidator.notice.DuplicatedColumnNotice; import org.mobilitydata.gtfsvalidator.notice.EmptyColumnNameNotice; +import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedColumnNotice; import org.mobilitydata.gtfsvalidator.notice.MissingRequiredColumnNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.UnknownColumnNotice; @@ -40,6 +42,7 @@ public void expectedColumns() { new CsvHeader(new String[] {"stop_id", "stop_name"}), ImmutableSet.of("stop_id", "stop_name", "stop_lat", "stop_lon"), ImmutableSet.of("stop_id"), + Set.of(), container); assertThat(container.getValidationNotices()).isEmpty(); @@ -55,6 +58,7 @@ public void unknownColumnShouldGenerateNotice() { new CsvHeader(new String[] {"stop_id", "stop_name", "stop_extra"}), ImmutableSet.of("stop_id", "stop_name"), ImmutableSet.of("stop_id"), + Set.of(), container); assertThat(container.getValidationNotices()) @@ -71,6 +75,7 @@ public void missingRequiredColumnShouldGenerateNotice() { new CsvHeader(new String[] {"stop_name"}), ImmutableSet.of("stop_id", "stop_name"), ImmutableSet.of("stop_id"), + Set.of(), container); assertThat(container.getValidationNotices()) @@ -78,6 +83,24 @@ public void missingRequiredColumnShouldGenerateNotice() { assertThat(container.hasValidationErrors()).isTrue(); } + @Test + public void missingRecommendedColumnShouldGenerateNotice() { + NoticeContainer container = new NoticeContainer(); + new DefaultTableHeaderValidator() + .validate( + "stops.txt", + new CsvHeader(new String[] {"stop_name"}), + Set.of("stop_id", "stop_name"), + Set.of(), + Set.of("stop_id"), + container); + + assertThat(container.getValidationNotices()) + .containsExactly(new MissingRecommendedColumnNotice("stops.txt", "stop_id")); + assertThat(container.hasValidationErrors()).isFalse(); + assertThat(container.hasValidationWarnings()).isTrue(); + } + @Test public void duplicatedColumnShouldGenerateNotice() { NoticeContainer container = new NoticeContainer(); @@ -87,6 +110,7 @@ public void duplicatedColumnShouldGenerateNotice() { new CsvHeader(new String[] {"stop_id", "stop_name", "stop_id"}), ImmutableSet.of("stop_id", "stop_name"), ImmutableSet.of("stop_id"), + Set.of(), container); assertThat(container.getValidationNotices()) @@ -103,6 +127,7 @@ public void emptyFileShouldNotGenerateNotice() { CsvHeader.EMPTY, ImmutableSet.of("stop_id", "stop_name"), ImmutableSet.of("stop_id"), + Set.of(), container); assertThat(container.getValidationNotices()).isEmpty(); @@ -118,6 +143,7 @@ public void emptyColumnNameShouldGenerateNotice() { new CsvHeader(new String[] {"stop_id", null, "stop_name", ""}), ImmutableSet.of("stop_id", "stop_name"), ImmutableSet.of("stop_id"), + Set.of(), container); assertThat(container.getValidationNotices()) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsStopTimeSchema.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsStopTimeSchema.java index 731479a1c2..f35ce24ff9 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsStopTimeSchema.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsStopTimeSchema.java @@ -29,6 +29,7 @@ import org.mobilitydata.gtfsvalidator.annotation.Index; import org.mobilitydata.gtfsvalidator.annotation.NonNegative; import org.mobilitydata.gtfsvalidator.annotation.PrimaryKey; +import org.mobilitydata.gtfsvalidator.annotation.RecommendedColumn; import org.mobilitydata.gtfsvalidator.annotation.Required; import org.mobilitydata.gtfsvalidator.type.GtfsTime; @@ -77,5 +78,6 @@ public interface GtfsStopTimeSchema extends GtfsEntity { double shapeDistTraveled(); @DefaultValue("1") + @RecommendedColumn GtfsStopTimeTimepoint timepoint(); } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidator.java index 4612667f20..dd81d5f72f 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidator.java @@ -22,6 +22,7 @@ import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.FileRefs; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; +import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedColumnNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.SeverityLevel; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; @@ -39,7 +40,6 @@ *

  • {@link StopTimeTimepointWithoutTimesNotice} - a timepoint does not specifies arrival_time * or departure_time *
  • {@link MissingTimepointValueNotice} - value for {@code stop_times.timepoint} is missing - *
  • {@link MissingTimepointColumnNotice} - field {@code stop_times.timepoint} is missing * */ @GtfsValidator @@ -59,7 +59,9 @@ public void validate(NoticeContainer noticeContainer) { // - this should be flagged; // - but also no notice regarding the absence of arrival_time or departure_time should be // generated - noticeContainer.addValidationNotice(new MissingTimepointColumnNotice()); + noticeContainer.addValidationNotice( + new MissingRecommendedColumnNotice( + GtfsStopTime.FILENAME, GtfsStopTime.TIMEPOINT_FIELD_NAME)); return; } for (GtfsStopTime stopTime : stopTimes.getEntities()) { @@ -142,20 +144,4 @@ static class MissingTimepointValueNotice extends ValidationNotice { this.stopSequence = stopTime.stopSequence(); } } - - /** `timepoint` column is missing for a dataset. */ - @GtfsValidationNotice( - severity = WARNING, - files = @FileRefs(GtfsStopTimeSchema.class), - bestPractices = @FileRefs(GtfsStopTimeSchema.class)) - static class MissingTimepointColumnNotice extends ValidationNotice { - - /** The name of the affected file. */ - private final String filename; - - MissingTimepointColumnNotice() { - super(SeverityLevel.WARNING); - this.filename = GtfsStopTime.FILENAME; - } - } } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidatorTest.java index 9e3a9aed2c..e1951da05c 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidatorTest.java @@ -33,13 +33,13 @@ import java.util.ArrayList; import java.util.List; import org.junit.Test; +import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedColumnNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.table.GtfsStopTime; import org.mobilitydata.gtfsvalidator.table.GtfsStopTimeTableContainer; import org.mobilitydata.gtfsvalidator.type.GtfsTime; -import org.mobilitydata.gtfsvalidator.validator.TimepointTimeValidator.MissingTimepointColumnNotice; import org.mobilitydata.gtfsvalidator.validator.TimepointTimeValidator.MissingTimepointValueNotice; import org.mobilitydata.gtfsvalidator.validator.TimepointTimeValidator.StopTimeTimepointWithoutTimesNotice; @@ -59,7 +59,8 @@ private static List generateNotices( return noticeContainer.getValidationNotices(); } - // using this header will trigger a MissingTimepointColumnNotice + // Using this header will trigger a MissingRecommendedColumnNotice since the timepoint column is + // missing. private static CsvHeader createLegacyHeader() { return new CsvHeader( new String[] { @@ -97,8 +98,8 @@ private static CsvHeader createHeaderWithTimepointColumn() { @Test public void noTimepointColumn_noTimeProvided_shouldGenerateNotice() { - // Using createLegacyHeader() that omits the timestamp column will trigger the - // MissingTimepointColumnNotice. + // Using createLegacyHeader() that omits the timepoint column will trigger the + // MissingRecommendedColumnNotice. // .setTimepoint(null) is used to indicate that no value is provided, although it has no effect // in this test. List stopTimes = new ArrayList<>(); @@ -123,7 +124,9 @@ public void noTimepointColumn_noTimeProvided_shouldGenerateNotice() { .setTimepoint((Integer) null) .build()); assertThat(generateNotices(createLegacyHeader(), stopTimes)) - .containsExactly(new MissingTimepointColumnNotice()); + .containsExactly( + new MissingRecommendedColumnNotice( + GtfsStopTime.FILENAME, GtfsStopTime.TIMEPOINT_FIELD_NAME)); } @Test @@ -144,7 +147,9 @@ public void noTimepointColumn_timesProvided_shouldGenerateNotice() { .setTimepoint((Integer) null) .build()); assertThat(generateNotices(createLegacyHeader(), stopTimes)) - .containsExactly(new MissingTimepointColumnNotice()); + .containsExactly( + new MissingRecommendedColumnNotice( + GtfsStopTime.FILENAME, GtfsStopTime.TIMEPOINT_FIELD_NAME)); } @Test diff --git a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/RecommendedColumn.java b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/RecommendedColumn.java new file mode 100644 index 0000000000..9b70ffcef8 --- /dev/null +++ b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/RecommendedColumn.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mobilitydata.gtfsvalidator.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Adds a validation that it's recommended that a column be present. A value for the field may be + * optional. + * + *

    Example. + * + * 

    {@code
+ * @GtfsTable("stop_times.txt")
+ * public interface GtfsStopTimeSchema extends GtfsEntity {
+ *   @FieldType(FieldTypeEnum.ID)
+ *   @PrimaryKey
+ *   @Index
+ *   @Required
+ *   @ForeignKey(table = "trips.txt", field = "trip_id")
+ *   String tripId();
+ *
+ *    ...
+ *
+ *   @DefaultValue("1")
+ *   @RecommendedColumn
+ *   GtfsStopTimeTimepoint timepoint();
+ * }
+ * }
    + */ +@Target({ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.SOURCE) +public @interface RecommendedColumn {} diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/Analyser.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/Analyser.java index f2d52b825c..0a15d5577b 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/Analyser.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/Analyser.java @@ -71,6 +71,7 @@ public GtfsFileDescriptor analyzeGtfsFileType(TypeElement type) { fieldBuilder.setRecommended(method.getAnnotation(Recommended.class) != null); fieldBuilder.setColumnRequired(method.getAnnotation(RequiredColumn.class) != null); fieldBuilder.setValueRequired(method.getAnnotation(Required.class) != null); + fieldBuilder.setColumnRecommended(method.getAnnotation(RecommendedColumn.class) != null); fieldBuilder.setMixedCase(method.getAnnotation(MixedCase.class) != null); PrimaryKey primaryKey = method.getAnnotation(PrimaryKey.class); if (primaryKey != null) { diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsFieldDescriptor.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsFieldDescriptor.java index d94b62c8ef..c5c6374fac 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsFieldDescriptor.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/GtfsFieldDescriptor.java @@ -59,6 +59,8 @@ public static GtfsFieldDescriptor.Builder builder() { public abstract boolean columnRequired(); + public abstract boolean columnRecommended(); + public boolean isHeaderRequired() { return columnRequired() || valueRequired(); } @@ -81,6 +83,8 @@ public abstract static class Builder { public abstract Builder setColumnRequired(boolean value); + public abstract Builder setColumnRecommended(boolean value); + public abstract Builder setMixedCase(boolean value); public abstract Builder setPrimaryKey(PrimaryKey annotation); diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java index cc96ce97da..72c57fdc89 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java @@ -184,12 +184,14 @@ private MethodSpec generateGetColumnsMethod() { "GtfsColumnDescriptor.builder()\n" + ".setColumnName($T.$L)\n" + ".setHeaderRequired($L)\n" + + ".setHeaderRecommended($L)\n" + ".setFieldLevel($T.$L)\n" + ".setIsMixedCase($L)\n" + ".setIsCached($L)\n", gtfsEntityType, fieldNameField(field.name()), field.isHeaderRequired(), + field.columnRecommended(), FieldLevelEnum.class, getFieldLevel(field), field.mixedCase(), diff --git a/processor/tests/src/main/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchema.java b/processor/tests/src/main/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchema.java new file mode 100644 index 0000000000..059a360b45 --- /dev/null +++ b/processor/tests/src/main/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchema.java @@ -0,0 +1,13 @@ +package org.mobilitydata.gtfsvalidator.processor.tests; + +import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; +import org.mobilitydata.gtfsvalidator.annotation.Recommended; + +@GtfsTable("recommended_column.txt") +public interface RecommendedColumnAnnotationSchema { + + @Recommended + String columnRecommended(); + + String valueNotRequired(); +} From 7ce3c113e8c28efe5a3a2bd844f005876d7d43ad Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 1 Jun 2023 16:57:53 -0400 Subject: [PATCH 2/7] Flagged to skip acceptance because this PR legitimately removes and adds a lot of notices [acceptance test skip] From 6b70c664d03ad52828c24bc6ea4dc43e45e5a9a6 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 1 Jun 2023 17:17:28 -0400 Subject: [PATCH 3/7] [acceptance test skip] Corrected some comments that aggregateJavadoc did not like. --- .../annotation/RecommendedColumn.java | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/RecommendedColumn.java b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/RecommendedColumn.java index 9b70ffcef8..f292ebd22d 100644 --- a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/RecommendedColumn.java +++ b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/RecommendedColumn.java @@ -27,23 +27,15 @@ * *

    Example. * - * 

    {@code
- * @GtfsTable("stop_times.txt")
- * public interface GtfsStopTimeSchema extends GtfsEntity {
- *   @FieldType(FieldTypeEnum.ID)
- *   @PrimaryKey
- *   @Index
- *   @Required
- *   @ForeignKey(table = "trips.txt", field = "trip_id")
- *   String tripId();
+ * 
    + *   {@literal @}GtfsTable("stop_times.txt")
+ *    public interface GtfsStopTimeSchema extends GtfsEntity {
  *
- *    ...
- *
- *   @DefaultValue("1")
- *   @RecommendedColumn
- *   GtfsStopTimeTimepoint timepoint();
- * }
- * }
    
+ *     {@literal @}DefaultValue("1")
+ *     {@literal @}RecommendedColumn
+ *      GtfsStopTimeTimepoint timepoint();
+ *   }
+ *
    */ @Target({ElementType.METHOD, ElementType.TYPE}) @Retention(RetentionPolicy.SOURCE) From fb8249238016b0a72416de42a0d5189d6d085f0f Mon Sep 17 00:00:00 2001 From: jcpitre Date: Mon, 5 Jun 2023 15:03:20 -0400 Subject: [PATCH 4/7] [acceptance test skip] Remove a useless method call that changed nothing to the execution. --- .../java/org/mobilitydata/gtfsvalidator/notice/Notice.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/Notice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/Notice.java index 43a50febb7..962973396a 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/Notice.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/Notice.java @@ -133,10 +133,10 @@ public boolean isError() { *

    This method is preferred to checking {@code severityLevel} directly since more levels may be * added in the future. * - * @return true if this notice is an warning, false otherwise + * @return true if this notice is a warning, false otherwise */ public boolean isWarning() { - return getSeverityLevel().ordinal() == SeverityLevel.WARNING.ordinal(); + return getSeverityLevel() == SeverityLevel.WARNING; } /** JSON exclusion strategy for notice context. It skips {@link Notice#severityLevel}. */ From 697d7d5a209930e53443932b554dcf33f9889e7e Mon Sep 17 00:00:00 2001 From: jcpitre Date: Tue, 20 Jun 2023 00:43:07 -0400 Subject: [PATCH 5/7] Corrected some merging errors and modified tests. --- .../MissingRecommendedColumnNotice.java | 1 - .../gtfsvalidator/notice/NoticeContainer.java | 2 +- .../gtfsvalidator/notice/ResolvedNotice.java | 12 ++++ .../validator/TimepointTimeValidator.java | 21 +------ .../validator/TimepointTimeValidatorTest.java | 57 ------------------- .../RecommendedColumnAnnotationSchema.java | 6 +- ...RecommendedColumnAnnotationSchemaTest.java | 51 +++++++++++++++++ 7 files changed, 67 insertions(+), 83 deletions(-) create mode 100644 processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchemaTest.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/MissingRecommendedColumnNotice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/MissingRecommendedColumnNotice.java index f40588daae..926186dbfc 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/MissingRecommendedColumnNotice.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/MissingRecommendedColumnNotice.java @@ -32,7 +32,6 @@ public class MissingRecommendedColumnNotice extends ValidationNotice { private final String fieldName; public MissingRecommendedColumnNotice(String filename, String fieldName) { - super(SeverityLevel.WARNING); this.filename = filename; this.fieldName = fieldName; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoticeContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoticeContainer.java index f7e7aaf4bc..c5231b5316 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoticeContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/NoticeContainer.java @@ -100,7 +100,7 @@ public void addValidationNoticeWithSeverity( if (resolved.isError()) { hasValidationErrors = true; } - if (resolved.isWarning()) ( + if (resolved.isWarning()) { hasValidationWarnings = true; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/ResolvedNotice.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/ResolvedNotice.java index feb26b2271..04a61be24d 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/ResolvedNotice.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/ResolvedNotice.java @@ -81,4 +81,16 @@ public int hashCode() { public boolean isError() { return getSeverityLevel().ordinal() >= SeverityLevel.ERROR.ordinal(); } + + /** + * Tells if this notice is a {@code WARNING}. + * + *

    This method is preferred to checking {@code severityLevel} directly since more levels may be + * added in the future. + * + * @return true if this notice is a warning, false otherwise + */ + public boolean isWarning() { + return getSeverityLevel() == SeverityLevel.WARNING; + } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidator.java index d9948204ed..972a35d8f0 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidator.java @@ -22,7 +22,6 @@ import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.FileRefs; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; -import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedColumnNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; import org.mobilitydata.gtfsvalidator.table.GtfsStopTime; @@ -55,12 +54,9 @@ public class TimepointTimeValidator extends FileValidator { public void validate(NoticeContainer noticeContainer) { if (!stopTimes.hasColumn(GtfsStopTime.TIMEPOINT_FIELD_NAME)) { // legacy datasets do not use timepoint column in stop_times.txt as a result: - // - this should be flagged; + // - this should be flagged in the header tests. // - but also no notice regarding the absence of arrival_time or departure_time should be // generated - noticeContainer.addValidationNotice( - new MissingRecommendedColumnNotice( - GtfsStopTime.FILENAME, GtfsStopTime.TIMEPOINT_FIELD_NAME)); return; } for (GtfsStopTime stopTime : stopTimes.getEntities()) { @@ -141,19 +137,4 @@ static class MissingTimepointValueNotice extends ValidationNotice { this.stopSequence = stopTime.stopSequence(); } } - - /** `timepoint` column is missing for a dataset. */ - @GtfsValidationNotice( - severity = WARNING, - files = @FileRefs(GtfsStopTimeSchema.class), - bestPractices = @FileRefs(GtfsStopTimeSchema.class)) - static class MissingTimepointColumnNotice extends ValidationNotice { - - /** The name of the affected file. */ - private final String filename; - - MissingTimepointColumnNotice() { - this.filename = GtfsStopTime.FILENAME; - } - } } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidatorTest.java index e1951da05c..159105a2af 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TimepointTimeValidatorTest.java @@ -33,7 +33,6 @@ import java.util.ArrayList; import java.util.List; import org.junit.Test; -import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedColumnNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; @@ -96,62 +95,6 @@ private static CsvHeader createHeaderWithTimepointColumn() { }); } - @Test - public void noTimepointColumn_noTimeProvided_shouldGenerateNotice() { - // Using createLegacyHeader() that omits the timepoint column will trigger the - // MissingRecommendedColumnNotice. - // .setTimepoint(null) is used to indicate that no value is provided, although it has no effect - // in this test. - List stopTimes = new ArrayList<>(); - stopTimes.add( - new GtfsStopTime.Builder() - .setCsvRowNumber(1) - .setTripId("first trip id") - .setArrivalTime(null) - .setDepartureTime(null) - .setStopId("stop id 0") - .setStopSequence(2) - .setTimepoint((Integer) null) - .build()); - stopTimes.add( - new GtfsStopTime.Builder() - .setCsvRowNumber(4) - .setTripId("second trip id") - .setArrivalTime(null) - .setDepartureTime(null) - .setStopId("stop id 1") - .setStopSequence(2) - .setTimepoint((Integer) null) - .build()); - assertThat(generateNotices(createLegacyHeader(), stopTimes)) - .containsExactly( - new MissingRecommendedColumnNotice( - GtfsStopTime.FILENAME, GtfsStopTime.TIMEPOINT_FIELD_NAME)); - } - - @Test - public void noTimepointColumn_timesProvided_shouldGenerateNotice() { - // Using createLegacyHeader() that omits the timestamp column will trigger the - // MissingTimepointColumnNotice. - // .setTimepoint(null) is used to indicate that no value is provided, although it has no effect - // in this test. - List stopTimes = new ArrayList<>(); - stopTimes.add( - new GtfsStopTime.Builder() - .setCsvRowNumber(1) - .setTripId("first trip id") - .setArrivalTime(GtfsTime.fromSecondsSinceMidnight(450)) - .setDepartureTime(GtfsTime.fromSecondsSinceMidnight(580)) - .setStopId("stop id") - .setStopSequence(2) - .setTimepoint((Integer) null) - .build()); - assertThat(generateNotices(createLegacyHeader(), stopTimes)) - .containsExactly( - new MissingRecommendedColumnNotice( - GtfsStopTime.FILENAME, GtfsStopTime.TIMEPOINT_FIELD_NAME)); - } - @Test public void timepointWithNoTimeShouldGenerateNotices() { List stopTimes = new ArrayList<>(); diff --git a/processor/tests/src/main/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchema.java b/processor/tests/src/main/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchema.java index 059a360b45..dff043ff42 100644 --- a/processor/tests/src/main/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchema.java +++ b/processor/tests/src/main/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchema.java @@ -1,13 +1,11 @@ package org.mobilitydata.gtfsvalidator.processor.tests; import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; -import org.mobilitydata.gtfsvalidator.annotation.Recommended; +import org.mobilitydata.gtfsvalidator.annotation.RecommendedColumn; @GtfsTable("recommended_column.txt") public interface RecommendedColumnAnnotationSchema { - @Recommended + @RecommendedColumn String columnRecommended(); - - String valueNotRequired(); } diff --git a/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchemaTest.java b/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchemaTest.java new file mode 100644 index 0000000000..fd600103e2 --- /dev/null +++ b/processor/tests/src/test/java/org/mobilitydata/gtfsvalidator/processor/tests/RecommendedColumnAnnotationSchemaTest.java @@ -0,0 +1,51 @@ +package org.mobilitydata.gtfsvalidator.processor.tests; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mobilitydata.gtfsvalidator.notice.MissingRequiredColumnNotice; +import org.mobilitydata.gtfsvalidator.table.RecommendedColumnAnnotationTableDescriptor; +import org.mobilitydata.gtfsvalidator.testing.LoadingHelper; +import org.mobilitydata.gtfsvalidator.validator.ValidatorLoaderException; + +@RunWith(JUnit4.class) +public class RecommendedColumnAnnotationSchemaTest { + private RecommendedColumnAnnotationTableDescriptor tableDescriptor; + private LoadingHelper helper; + + @Before + public void setup() throws ValidatorLoaderException { + tableDescriptor = new RecommendedColumnAnnotationTableDescriptor(); + helper = new LoadingHelper(); + } + + @Test + public void includingRecommendedColumnHeaderWithoutValueShouldNotGenerateNotice() + throws ValidatorLoaderException { + + helper.load(tableDescriptor, "some_column,column_recommended", "value,"); + + assertThat( + !helper + .getValidationNotices() + .contains( + new MissingRequiredColumnNotice("recommended_column.txt", "column_recommended"))); + } + + @Test + public void missingRecommendedColumnHeaderShouldGenerateNotice() throws ValidatorLoaderException { + + helper.load(tableDescriptor, "column", "value"); + // Since we use an unknown column ("column") we have to expect at least one unknown_column + // notice along with the + // missing_recommended_column notice. + assertThat( + helper + .getValidationNotices() + .contains( + new MissingRequiredColumnNotice("recommended_column.txt", "column_recommended"))); + } +} From d8139d6fb9682db243813dbd3a8cf87ae9a37ff0 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 21 Jun 2023 11:28:45 -0400 Subject: [PATCH 6/7] Corrected according to PR comments. --- .../gtfsvalidator/validator/TableHeaderValidatorTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidatorTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidatorTest.java index 4dfd474832..1946ecd024 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidatorTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/TableHeaderValidatorTest.java @@ -80,7 +80,6 @@ public void missingRequiredColumnShouldGenerateNotice() { assertThat(container.getValidationNotices()) .containsExactly(new MissingRequiredColumnNotice("stops.txt", "stop_id")); - assertThat(container.hasValidationErrors()).isTrue(); } @Test @@ -97,8 +96,6 @@ public void missingRecommendedColumnShouldGenerateNotice() { assertThat(container.getValidationNotices()) .containsExactly(new MissingRecommendedColumnNotice("stops.txt", "stop_id")); - assertThat(container.hasValidationErrors()).isFalse(); - assertThat(container.hasValidationWarnings()).isTrue(); } @Test From ca8dba8794f59d85eafa3f085837c0f1fc53327a Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 21 Jun 2023 13:55:21 -0400 Subject: [PATCH 7/7] Repaired test broken after merge of master --- .../mobilitydata/gtfsvalidator/parsing/RowParserTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/parsing/RowParserTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/parsing/RowParserTest.java index c3c0db2cf7..260f3da2e2 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/parsing/RowParserTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/parsing/RowParserTest.java @@ -56,6 +56,11 @@ public boolean headerRequired() { return false; } + @Override + public boolean headerRecommended() { + return false; + } + @Override public FieldLevelEnum fieldLevel() { return FieldLevelEnum.REQUIRED; @@ -141,6 +146,7 @@ public void asString_recommended_missing() { GtfsColumnDescriptor.builder() .setColumnName("column name") .setHeaderRequired(false) + .setHeaderRecommended(false) .setFieldLevel(FieldLevelEnum.RECOMMENDED) .setIsMixedCase(false) .setIsCached(false)