diff --git a/drape_frontend/color_constants.cpp b/drape_frontend/color_constants.cpp index f7fff950634..c3591e24c96 100644 --- a/drape_frontend/color_constants.cpp +++ b/drape_frontend/color_constants.cpp @@ -107,10 +107,6 @@ TransitColorsHolder & TransitColors() namespace df { -string const kTransitColorPrefix = "transit_"; -string const kTransitTextPrefix = "text_"; -string const kTransitLinePrefix = "line_"; - ColorConstant GetTransitColorName(ColorConstant const & localName) { return kTransitColorPrefix + kTransitLinePrefix + localName; @@ -134,10 +130,10 @@ dp::Color GetColorConstant(ColorConstant const & constant) return ToDrapeColor(color); } +map const & GetTransitClearColors() { return TransitColors().GetClearColors(); } + void LoadTransitColors() { TransitColors().Load(); } - -map const & GetClearColors() { return TransitColors().GetClearColors(); } } // namespace df diff --git a/drape_frontend/color_constants.hpp b/drape_frontend/color_constants.hpp index 13d311f5e68..6a3f2890c60 100644 --- a/drape_frontend/color_constants.hpp +++ b/drape_frontend/color_constants.hpp @@ -9,8 +9,12 @@ namespace df { using ColorConstant = std::string; +inline std::string const kTransitColorPrefix = "transit_"; +inline std::string const kTransitTextPrefix = "text_"; +inline std::string const kTransitLinePrefix = "line_"; + dp::Color GetColorConstant(ColorConstant const & constant); -std::map const & GetClearColors(); +std::map const & GetTransitClearColors(); void LoadTransitColors(); ColorConstant GetTransitColorName(ColorConstant const & localName); diff --git a/transit/CMakeLists.txt b/transit/CMakeLists.txt index f4f2e6b7c98..a047b362b94 100644 --- a/transit/CMakeLists.txt +++ b/transit/CMakeLists.txt @@ -17,3 +17,4 @@ set( omim_add_library(${PROJECT_NAME} ${SRC}) omim_add_test_subdirectory(transit_tests) +add_subdirectory(world_feed) diff --git a/transit/world_feed/CMakeLists.txt b/transit/world_feed/CMakeLists.txt new file mode 100644 index 00000000000..f7675a3b41f --- /dev/null +++ b/transit/world_feed/CMakeLists.txt @@ -0,0 +1,73 @@ +project(world_feed) + +set(SRC + color_picker.cpp + color_picker.hpp + date_time_helpers.cpp + date_time_helpers.hpp + feed_helpers.cpp + feed_helpers.hpp + world_feed.cpp + world_feed.hpp +) + +omim_add_library(${PROJECT_NAME} ${SRC}) + +omim_link_libraries( + ${PROJECT_NAME} + drape_frontend + shaders + routing + mwm_diff + bsdiff + tracking + traffic + routing_common + transit + descriptions + ugc + drape + partners_api + web_api + local_ads + kml + editor + indexer + metrics + platform + geometry + coding + base + freetype + expat + icu + agg + jansson + protobuf + stats_client + minizip + succinct + pugixml + oauthcpp + opening_hours + stb_image + sdf_image + vulkan_wrapper + ${Qt5Widgets_LIBRARIES} + ${Qt5Network_LIBRARIES} + ${LIBZ} +) + +if (PLATFORM_LINUX) + omim_link_libraries( + ${PROJECT_NAME} + dl + ) +endif() + +link_opengl(${PROJECT_NAME}) +link_qt5_core(${PROJECT_NAME}) +link_qt5_network(${PROJECT_NAME}) + +omim_add_test_subdirectory(world_feed_tests) +add_subdirectory(gtfs_converter) diff --git a/transit/world_feed/color_picker.cpp b/transit/world_feed/color_picker.cpp new file mode 100644 index 00000000000..94bf13f5071 --- /dev/null +++ b/transit/world_feed/color_picker.cpp @@ -0,0 +1,69 @@ +#include "transit/world_feed/color_picker.hpp" + +#include "drape_frontend/apply_feature_functors.hpp" +#include "drape_frontend/color_constants.hpp" + +#include "drape/color.hpp" + +#include "base/string_utils.hpp" + +#include +#include + +namespace +{ +std::tuple GetColors(dp::Color const & color) +{ + return {color.GetRedF(), color.GetGreenF(), color.GetBlueF()}; +} + +double GetSquareDistance(dp::Color const & color1, dp::Color const & color2) +{ + auto [r1, g1, b1] = GetColors(color1); + auto [r2, g2, b2] = GetColors(color2); + return (r1 - r2) * (r1 - r2) + (g1 - g2) * (g1 - g2) + (b1 - b2) * (b1 - b2); +} +} // namespace + +namespace transit +{ +ColorPicker::ColorPicker() { df::LoadTransitColors(); } + +std::string ColorPicker::GetNearestColor(std::string const & rgb) +{ + static std::string const kDefaultColor = "default"; + if (rgb.empty()) + return kDefaultColor; + + auto [it, inserted] = m_colorsToNames.emplace(rgb, kDefaultColor); + if (!inserted) + return it->second; + + std::string nearestColor = kDefaultColor; + + unsigned int intColor; + // We do not need to add to the cache invalid color, so we just return. + if (!strings::to_uint(rgb, intColor, 16)) + return nearestColor; + + dp::Color const color = df::ToDrapeColor(static_cast(intColor)); + double minDist = std::numeric_limits::max(); + + for (auto const & [name, transitColor] : df::GetTransitClearColors()) + { + if (double const dist = GetSquareDistance(color, transitColor); dist < minDist) + { + minDist = dist; + nearestColor = name; + } + } + if (nearestColor.find(df::kTransitColorPrefix + df::kTransitLinePrefix) == 0) + { + nearestColor = + nearestColor.substr(df::kTransitColorPrefix.size() + df::kTransitLinePrefix.size()); + } + + it->second = nearestColor; + return nearestColor; +} +} // namespace transit diff --git a/transit/world_feed/color_picker.hpp b/transit/world_feed/color_picker.hpp new file mode 100644 index 00000000000..7e5c2c72945 --- /dev/null +++ b/transit/world_feed/color_picker.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +namespace transit +{ +class ColorPicker +{ +public: + ColorPicker(); + std::string GetNearestColor(std::string const & rgb); + +private: + std::unordered_map m_colorsToNames; +}; +} // namespace transit diff --git a/transit/world_feed/date_time_helpers.cpp b/transit/world_feed/date_time_helpers.cpp new file mode 100644 index 00000000000..be290f15f1c --- /dev/null +++ b/transit/world_feed/date_time_helpers.cpp @@ -0,0 +1,265 @@ +#include "transit/world_feed/date_time_helpers.hpp" + +#include "base/assert.hpp" +#include "base/logging.hpp" + +#include +#include +#include +#include + +#include "3party/boost/boost/date_time/gregorian/gregorian.hpp" + +namespace transit +{ +osmoh::Time GetTimeOsmoh(gtfs::Time const & gtfsTime) +{ + uint16_t hh; + uint16_t mm; + std::tie(hh, mm, std::ignore) = gtfsTime.get_hh_mm_ss(); + return osmoh::Time(osmoh::Time::THours(hh) + osmoh::Time::TMinutes(mm)); +} + +osmoh::RuleSequence GetRuleSequenceOsmoh(gtfs::Time const & start, gtfs::Time const & end) +{ + osmoh::RuleSequence ruleSeq; + ruleSeq.SetModifier(osmoh::RuleSequence::Modifier::Open); + osmoh::Timespan range(GetTimeOsmoh(start), GetTimeOsmoh(end)); + ruleSeq.SetTimes({range}); + return ruleSeq; +} + +osmoh::MonthdayRange GetMonthdayRangeFromDates(gtfs::Date const & start, gtfs::Date const & end) +{ + osmoh::MonthdayRange range; + SetOpeningHoursRange(range, start, true /* isStart */); + SetOpeningHoursRange(range, end, false /* isStart */); + return range; +} + +struct AccumExceptionDates +{ +public: + using GregorianInterval = std::pair; + using GtfsInterval = std::pair; + + void InitIntervals(boost::gregorian::date const & gregorianDate, gtfs::Date const & gtfsDate); + void AddRange(); + bool IsInited() const; + + GregorianInterval m_GregorianInterval; + GtfsInterval m_GtfsInterval; + osmoh::TMonthdayRanges m_ranges; + +private: + bool m_inited = false; +}; + +void AccumExceptionDates::InitIntervals(boost::gregorian::date const & gregorianDate, + gtfs::Date const & gtfsDate) +{ + m_GregorianInterval = std::make_pair(gregorianDate, gregorianDate); + m_GtfsInterval = std::make_pair(gtfsDate, gtfsDate); + m_inited = true; +} + +void AccumExceptionDates::AddRange() +{ + osmoh::MonthdayRange range = + GetMonthdayRangeFromDates(m_GtfsInterval.first, m_GtfsInterval.second); + m_ranges.push_back(range); + m_inited = false; +} + +bool AccumExceptionDates::IsInited() const { return m_inited; } + +std::string ToString(osmoh::OpeningHours const & openingHours) +{ + if (!openingHours.IsValid()) + return {}; + + std::ostringstream stream; + stream << openingHours.GetRule(); + return stream.str(); +} + +osmoh::Weekday ConvertWeekDayIndexToOsmoh(size_t index) +{ + // Monday index in osmoh is 2. + index += 2; + + if (index == 7) + return osmoh::Weekday::Saturday; + if (index == 8) + return osmoh::Weekday::Sunday; + + return osmoh::ToWeekday(index); +} + +std::vector GetOpenCloseIntervals( + std::vector const & week) +{ + std::vector intervals; + + WeekdaysInterval interval; + for (size_t i = 0; i < week.size(); ++i) + { + osmoh::RuleSequence::Modifier const status = week[i] == gtfs::CalendarAvailability::Available + ? osmoh::RuleSequence::Modifier::DefaultOpen + : osmoh::RuleSequence::Modifier::Closed; + if (status == interval.m_status) + { + interval.m_end = i; + } + else + { + if (i > 0) + intervals.push_back(interval); + interval.m_start = i; + interval.m_end = i; + interval.m_status = status; + } + if (i == week.size() - 1) + intervals.push_back(interval); + } + + return intervals; +} + +void SetOpeningHoursRange(osmoh::MonthdayRange & range, gtfs::Date const & date, bool isStart) +{ + if (!date.is_provided()) + { + LOG(LINFO, ("Date is not provided in the calendar.")); + return; + } + + auto const & [year, month, day] = date.get_yyyy_mm_dd(); + + osmoh::MonthDay monthDay; + monthDay.SetYear(year); + monthDay.SetMonth(static_cast(month)); + monthDay.SetDayNum(day); + + if (isStart) + range.SetStart(monthDay); + else + range.SetEnd(monthDay); +} + +void GetServiceDaysOsmoh(gtfs::CalendarItem const & serviceDays, osmoh::TRuleSequences & rules) +{ + osmoh::MonthdayRange range = + GetMonthdayRangeFromDates(serviceDays.start_date, serviceDays.end_date); + osmoh::TMonthdayRanges const rangesMonths{range}; + + std::vector const weekDayStatuses = { + serviceDays.monday, serviceDays.tuesday, serviceDays.wednesday, serviceDays.thursday, + serviceDays.friday, serviceDays.saturday, serviceDays.sunday}; + + auto const & intervals = GetOpenCloseIntervals(weekDayStatuses); + + osmoh::RuleSequence ruleSeqOpen; + osmoh::RuleSequence ruleSeqClose; + + for (auto const & interval : intervals) + { + osmoh::RuleSequence & ruleSeq = interval.m_status == osmoh::RuleSequence::Modifier::DefaultOpen + ? ruleSeqOpen + : ruleSeqClose; + ruleSeq.SetMonths(rangesMonths); + ruleSeq.SetModifier(interval.m_status); + + osmoh::WeekdayRange weekDayRange; + weekDayRange.SetStart(ConvertWeekDayIndexToOsmoh(interval.m_start)); + weekDayRange.SetEnd(ConvertWeekDayIndexToOsmoh(interval.m_end)); + + osmoh::TWeekdayRanges weekDayRanges; + weekDayRanges.push_back(weekDayRange); + + osmoh::Weekdays weekDays; + weekDays.SetWeekdayRanges(weekDayRanges); + ruleSeq.SetWeekdays(weekDays); + } + + if (ruleSeqOpen.HasWeekdays()) + rules.push_back(ruleSeqOpen); + + if (ruleSeqClose.HasWeekdays()) + rules.push_back(ruleSeqClose); +} + +void AppendMonthRules(osmoh::RuleSequence::Modifier const & status, + osmoh::TMonthdayRanges const & monthRanges, osmoh::TRuleSequences & rules) +{ + osmoh::RuleSequence ruleSeq; + ruleSeq.SetMonths(monthRanges); + ruleSeq.SetModifier(status); + rules.push_back(ruleSeq); +} + +void GetServiceDaysExceptionsOsmoh(gtfs::CalendarDates const & exceptionDays, + osmoh::TRuleSequences & rules) +{ + if (exceptionDays.empty()) + return; + + AccumExceptionDates accumOpen; + AccumExceptionDates accumClosed; + + for (size_t i = 0; i < exceptionDays.size(); ++i) + { + AccumExceptionDates & curAccum = + (exceptionDays[i].exception_type == gtfs::CalendarDateException::Added) ? accumOpen + : accumClosed; + + auto const [year, month, day] = exceptionDays[i].date.get_yyyy_mm_dd(); + boost::gregorian::date const date{year, month, day}; + if (!curAccum.IsInited()) + { + curAccum.InitIntervals(date, exceptionDays[i].date); + } + else + { + auto & prevDate = curAccum.m_GregorianInterval.second; + boost::gregorian::date_duration duration = date - prevDate; + CHECK(!duration.is_negative(), ()); + + if (duration.days() == 1) + { + prevDate = date; + curAccum.m_GtfsInterval.second = exceptionDays[i].date; + } + else + { + curAccum.AddRange(); + curAccum.InitIntervals(date, exceptionDays[i].date); + } + } + + AccumExceptionDates & prevAccum = + (exceptionDays[i].exception_type == gtfs::CalendarDateException::Added) ? accumClosed + : accumOpen; + if (prevAccum.IsInited()) + prevAccum.AddRange(); + + if (i == exceptionDays.size() - 1) + curAccum.AddRange(); + } + + if (!accumOpen.m_ranges.empty()) + AppendMonthRules(osmoh::RuleSequence::Modifier::DefaultOpen, accumOpen.m_ranges, rules); + + if (!accumClosed.m_ranges.empty()) + AppendMonthRules(osmoh::RuleSequence::Modifier::Closed, accumClosed.m_ranges, rules); +} + +void MergeRules(osmoh::TRuleSequences & dstRules, osmoh::TRuleSequences const & srcRules) +{ + for (auto const & rule : srcRules) + { + if (std::find(dstRules.begin(), dstRules.end(), rule) == dstRules.end()) + dstRules.push_back(rule); + } +} +} // namespace transit diff --git a/transit/world_feed/date_time_helpers.hpp b/transit/world_feed/date_time_helpers.hpp new file mode 100644 index 00000000000..51322b524e7 --- /dev/null +++ b/transit/world_feed/date_time_helpers.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include +#include + +#include "3party/just_gtfs/just_gtfs.h" +#include "3party/opening_hours/opening_hours.hpp" + +namespace transit +{ +// Creates osmoh::Time object from GTFS Time |gtfsTime|. +osmoh::Time GetTimeOsmoh(gtfs::Time const & gtfsTime); + +// Creates osmoh::RuleSequence with Modifier::Open and osmoh::Timespan with |start| - |end| +// interval. +osmoh::RuleSequence GetRuleSequenceOsmoh(gtfs::Time const & start, gtfs::Time const & end); + +// Converts |openingHours| to string. +std::string ToString(osmoh::OpeningHours const & openingHours); + +// Converts week day |index| in range [0, 6] to the osmoh::Weekday object. +osmoh::Weekday ConvertWeekDayIndexToOsmoh(size_t index); + +// Inclusive interval of days and corresponding Open/Closed status. +struct WeekdaysInterval +{ + size_t m_start = 0; + size_t m_end = 0; + osmoh::RuleSequence::Modifier m_status = osmoh::RuleSequence::Modifier::DefaultOpen; +}; + +// Calculates open/closed intervals for |week|. +std::vector GetOpenCloseIntervals( + std::vector const & week); + +// Sets start or end |date| for |range|. +void SetOpeningHoursRange(osmoh::MonthdayRange & range, gtfs::Date const & date, bool isStart); + +// Extracts open/closed service days ranges from |serviceDays| to |rules|. +void GetServiceDaysOsmoh(gtfs::CalendarItem const & serviceDays, osmoh::TRuleSequences & rules); + +// Extracts open/closed exception service days ranges from |exceptionDays| to |rules|. +void GetServiceDaysExceptionsOsmoh(gtfs::CalendarDates const & exceptionDays, + osmoh::TRuleSequences & rules); + +// Adds |srcRules| to |dstRules| if they are not present. +void MergeRules(osmoh::TRuleSequences & dstRules, osmoh::TRuleSequences const & srcRules); +} // namespace transit diff --git a/transit/world_feed/feed_helpers.cpp b/transit/world_feed/feed_helpers.cpp new file mode 100644 index 00000000000..aebbb5d6681 --- /dev/null +++ b/transit/world_feed/feed_helpers.cpp @@ -0,0 +1,265 @@ +#include "transit/world_feed/feed_helpers.hpp" + +#include "geometry/mercator.hpp" +#include "geometry/parametrized_segment.hpp" +#include "geometry/point2d.hpp" + +#include "base/assert.hpp" +#include "base/logging.hpp" + +#include +#include + +namespace +{ +// Epsilon for m2::PointD comparison. +double constexpr kEps = 1e-5; + +struct ProjectionData +{ + // Projection to polyline. + m2::PointD m_proj; + // Index before which the projection will be inserted. + size_t m_indexOnShape = 0; + // Distance from point to its projection. + double m_distFromPoint = 0.0; + // Distance from start point on polyline to the projection. + double m_distFromStart = 0.0; + // Point on polyline almost equal to the projection can already exist, so we don't need to + // insert projection. Or we insert it to the polyline. + bool m_needsInsertion = false; +}; + +// Returns true if |p1| is much closer to start then |p2| (parameter |distDeltaStart|) and its +// distance to projections to polyline |m_distFromPoint| is comparable. +bool CloserToStartAndOnSimilarDistToLine(ProjectionData const & p1, ProjectionData const & p2) +{ + // Delta between two points distances from start point on polyline. + double constexpr distDeltaStart = 100.0; + // Delta between two points distances from their corresponding projections to polyline. + double constexpr distDeltaProj = 90.0; + + return (p1.m_distFromStart + distDeltaStart < p2.m_distFromStart && + std::abs(p2.m_distFromPoint - p1.m_distFromPoint) <= distDeltaProj); +} +} // namespace + +namespace transit +{ +ProjectionToShape ProjectStopOnTrack(m2::PointD const & stopPoint, m2::PointD const & point1, + m2::PointD const & point2) +{ + m2::PointD const stopProjection = + m2::ParametrizedSegment(point1, point2).ClosestPointTo(stopPoint); + double const distM = mercator::DistanceOnEarth(stopProjection, stopPoint); + return {stopProjection, distM}; +} + +ProjectionData GetProjection(std::vector const & polyline, size_t index, + ProjectionToShape const & proj) +{ + ProjectionData projData; + projData.m_distFromPoint = proj.m_dist; + projData.m_proj = proj.m_point; + + if (base::AlmostEqualAbs(proj.m_point, polyline[index], kEps)) + { + projData.m_indexOnShape = index; + projData.m_needsInsertion = false; + } + else if (base::AlmostEqualAbs(proj.m_point, polyline[index + 1], kEps)) + { + projData.m_indexOnShape = index + 1; + projData.m_needsInsertion = false; + } + else + { + projData.m_indexOnShape = index + 1; + projData.m_needsInsertion = true; + } + + return projData; +} + +void FillProjections(std::vector & polyline, size_t startIndex, size_t endIndex, + m2::PointD const & point, std::vector & projections) +{ + double distTravelledM = 0.0; + // Stop can't be further from its projection to line then |maxDistFromStopM|. + double constexpr maxDistFromStopM = 1000; + + for (size_t i = startIndex; i < endIndex; ++i) + { + if (i > startIndex) + distTravelledM += mercator::DistanceOnEarth(polyline[i - 1], polyline[i]); + + auto proj = GetProjection(polyline, i, ProjectStopOnTrack(point, polyline[i], polyline[i + 1])); + proj.m_distFromStart = distTravelledM + mercator::DistanceOnEarth(polyline[i], proj.m_proj); + + if (proj.m_distFromPoint < maxDistFromStopM) + projections.emplace_back(proj); + } +} + +std::pair PrepareNearestPointOnTrack(m2::PointD const & point, size_t startIndex, + std::vector & polyline) +{ + std::vector projections; + // Reserve space for points on polyline which are relatively close to the polyline. + // Approximately 1/4 of all points on shape. + projections.reserve(polyline.size() / 4); + + FillProjections(polyline, startIndex, polyline.size() - 1, point, projections); + + if (projections.empty()) + return {polyline.size() + 1, false}; + + // We find the most fitting projection of the stop to the polyline. For two different projections + // with approximately equal distances to the stop the most preferable is the one that is closer + // to the beginning of the polyline segment. + auto const proj = + std::min_element(projections.begin(), projections.end(), + [](ProjectionData const & p1, ProjectionData const & p2) { + if (CloserToStartAndOnSimilarDistToLine(p1, p2)) + return true; + + if (CloserToStartAndOnSimilarDistToLine(p2, p1)) + return false; + + if (base::AlmostEqualAbs(p1.m_distFromPoint, p2.m_distFromPoint, kEps)) + return p1.m_distFromStart < p2.m_distFromStart; + + return p1.m_distFromPoint < p2.m_distFromPoint; + }); + + if (proj->m_needsInsertion) + polyline.insert(polyline.begin() + proj->m_indexOnShape, proj->m_proj); + + return {proj->m_indexOnShape, proj->m_needsInsertion}; +} + +bool IsRelevantType(const gtfs::RouteType & routeType) +{ + // All types and constants are described in GTFS: + // https://developers.google.com/transit/gtfs/reference + + // We skip all subways because we extract subway data from OSM, not from GTFS. + if (routeType == gtfs::RouteType::Subway) + return false; + + auto const val = static_cast(routeType); + // "Classic" GTFS route types. + if (val < 8 || (val > 10 && val < 13)) + return true; + + // Extended GTFS route types. + // We do not handle taxi services. + if (val >= 1500) + return false; + + // Other not relevant types - school buses, lorry services etc. + static std::vector const kNotRelevantTypes{ + gtfs::RouteType::CarTransportRailService, + gtfs::RouteType::LorryTransportRailService, + gtfs::RouteType::VehicleTransportRailService, + gtfs::RouteType::MetroService, + gtfs::RouteType::UndergroundService, + gtfs::RouteType::PostBusService, + gtfs::RouteType::SpecialNeedsBus, + gtfs::RouteType::MobilityBusService, + gtfs::RouteType::MobilityBusForRegisteredDisabled, + gtfs::RouteType::SchoolBus, + gtfs::RouteType::SchoolAndPublicServiceBus}; + + return std::find(kNotRelevantTypes.begin(), kNotRelevantTypes.end(), routeType) == + kNotRelevantTypes.end(); +} + +std::string ToString(gtfs::RouteType const & routeType) +{ + // GTFS route types. + switch (routeType) + { + case gtfs::RouteType::Tram: return "tram"; + case gtfs::RouteType::Subway: return "subway"; + case gtfs::RouteType::Rail: return "rail"; + case gtfs::RouteType::Bus: return "bus"; + case gtfs::RouteType::Ferry: return "ferry"; + case gtfs::RouteType::CableTram: return "cable_tram"; + case gtfs::RouteType::AerialLift: return "aerial_lift"; + case gtfs::RouteType::Funicular: return "funicular"; + case gtfs::RouteType::Trolleybus: return "trolleybus"; + case gtfs::RouteType::Monorail: return "monorail"; + default: + // Extended GTFS route types. + return ToStringExtendedType(routeType); + } +} + +std::string ToStringExtendedType(gtfs::RouteType const & routeType) +{ + // These constants refer to extended GTFS routes types. + auto const val = static_cast(routeType); + if (val >= 100 && val < 200) + return "rail"; + + if (val >= 200 && val < 300) + return "bus"; + + if (val == 405) + return "monorail"; + + if (val >= 400 && val < 500) + return "rail"; + + if (val >= 700 && val < 800) + return "bus"; + + if (val == 800) + return "trolleybus"; + + if (val >= 900 && val < 1000) + return "tram"; + + if (val == 1000) + return "water_service"; + + if (val == 1100) + return "air_service"; + + if (val == 1200) + return "ferry"; + + if (val == 1300) + return "aerial_lift"; + + if (val == 1400) + return "funicular"; + + LOG(LINFO, ("Unrecognized route type", val)); + return {}; +} + +gtfs::StopTimes GetStopTimesForTrip(gtfs::StopTimes const & allStopTimes, + std::string const & tripId) +{ + gtfs::StopTime reference; + reference.trip_id = tripId; + + auto itStart = std::lower_bound( + allStopTimes.begin(), allStopTimes.end(), reference, + [](const gtfs::StopTime & t1, const gtfs::StopTime & t2) { return t1.trip_id < t2.trip_id; }); + + if (itStart == allStopTimes.end()) + return {}; + auto itEnd = itStart; + while (itEnd != allStopTimes.end() && itEnd->trip_id == tripId) + ++itEnd; + + gtfs::StopTimes res(itStart, itEnd); + std::sort(res.begin(), res.end(), [](gtfs::StopTime const & t1, gtfs::StopTime const & t2) { + return t1.stop_sequence < t2.stop_sequence; + }); + return res; +} +} // namespace transit diff --git a/transit/world_feed/feed_helpers.hpp b/transit/world_feed/feed_helpers.hpp new file mode 100644 index 00000000000..8ae171483dc --- /dev/null +++ b/transit/world_feed/feed_helpers.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "geometry/point2d.hpp" + +#include +#include +#include + +#include "3party/just_gtfs/just_gtfs.h" + +namespace transit +{ +// Projection point and mercator distance to it. +struct ProjectionToShape +{ + m2::PointD m_point; + double m_dist; +}; + +/// \returns |stopPoint| projection to the track segment [|point1|, |point2|] and +/// distance from the |stopPoint| to its projection. +ProjectionToShape ProjectStopOnTrack(m2::PointD const & stopPoint, m2::PointD const & point1, + m2::PointD const & point2); + +/// \returns index of the nearest track point to the |point| and flag if it was inserted to the +/// shape. If this index doesn't match already existent points, the stop projection is inserted to +/// the |polyline| and the flag is set to true. +std::pair PrepareNearestPointOnTrack(m2::PointD const & point, size_t startIndex, + std::vector & polyline); + +/// \returns true if we should not skip routes with this GTFS |routeType|. +bool IsRelevantType(const gtfs::RouteType & routeType); + +/// \return string representation of the GTFS |routeType|. +std::string ToString(gtfs::RouteType const & routeType); + +/// \return string representation of the extended GTFS |routeType|. +std::string ToStringExtendedType(gtfs::RouteType const & routeType); + +/// \return stop times for trip with |tripId|. +gtfs::StopTimes GetStopTimesForTrip(gtfs::StopTimes const & allStopTimes, + std::string const & tripId); +} // namespace transit diff --git a/transit/world_feed/gtfs_converter/CMakeLists.txt b/transit/world_feed/gtfs_converter/CMakeLists.txt new file mode 100644 index 00000000000..32c56dc858a --- /dev/null +++ b/transit/world_feed/gtfs_converter/CMakeLists.txt @@ -0,0 +1,12 @@ +project(gtfs_converter) + +include_directories(${OMIM_ROOT}/3party/gflags/src) + +omim_add_executable(${PROJECT_NAME} + gtfs_converter.cpp) + +omim_link_libraries( + ${PROJECT_NAME} + world_feed + gflags +) diff --git a/transit/world_feed/gtfs_converter/gtfs_converter.cpp b/transit/world_feed/gtfs_converter/gtfs_converter.cpp new file mode 100644 index 00000000000..a62300a875c --- /dev/null +++ b/transit/world_feed/gtfs_converter/gtfs_converter.cpp @@ -0,0 +1,266 @@ +#include "transit/world_feed/color_picker.hpp" +#include "transit/world_feed/world_feed.hpp" + +#include "platform/platform.hpp" + +#include "base/assert.hpp" +#include "base/file_name_utils.hpp" +#include "base/logging.hpp" +#include "base/timer.hpp" + +#include "3party/gflags/src/gflags/gflags.h" + +DEFINE_string(path_mapping, "", "Path to the mapping file of TransitId to GTFS hash"); +DEFINE_string(path_gtfs_feeds, "", "Directory with GTFS feeds subdirectories"); +DEFINE_string(path_json, "", "Output directory for dumping json files"); +DEFINE_string(path_resources, "", "MAPS.ME resources directory"); +DEFINE_string(start_feed, "", "Optional. Feed directory from which the process continues"); +DEFINE_string(stop_feed, "", "Optional. Feed directory on which to stop the process"); + +// Finds subdirectories with feeds. +Platform::FilesList GetGtfsFeedsInDirectory(std::string const & path) +{ + Platform::FilesList res; + Platform::TFilesWithType gtfsList; + Platform::GetFilesByType(path, Platform::FILE_TYPE_DIRECTORY, gtfsList); + + for (auto const & item : gtfsList) + { + auto const & gtfsFeedDir = item.first; + if (gtfsFeedDir != "." && gtfsFeedDir != "..") + res.push_back(base::JoinPath(path, gtfsFeedDir)); + } + + return res; +} + +// Handles the case when the directory consists of a single subdirectory with GTFS files. +void ExtendPath(std::string & path) +{ + Platform::TFilesWithType csvFiles; + Platform::GetFilesByType(path, Platform::FILE_TYPE_REGULAR, csvFiles); + if (!csvFiles.empty()) + return; + + Platform::TFilesWithType subdirs; + Platform::GetFilesByType(path, Platform::FILE_TYPE_DIRECTORY, subdirs); + + // If there are more subdirectories then ".", ".." and directory with feed, the feed is most + // likely corrupted. + if (subdirs.size() > 3) + return; + + for (auto const & item : subdirs) + { + auto const & subdir = item.first; + if (subdir != "." && subdir != "..") + { + path = base::JoinPath(path, subdir); + LOG(LDEBUG, ("Found subdirectory with feed", path)); + return; + } + } +} + +bool SkipFeed(std::string const & feedPath, bool & pass) +{ + if (!FLAGS_start_feed.empty() && pass) + { + if (base::GetNameFromFullPath(feedPath) != FLAGS_start_feed) + return true; + pass = false; + } + return false; +} + +bool StopOnFeed(std::string const & feedPath) +{ + if (!FLAGS_stop_feed.empty() && base::GetNameFromFullPath(feedPath) == FLAGS_stop_feed) + { + LOG(LINFO, ("Stop on", feedPath)); + return true; + } + return false; +} + +enum class FeedStatus +{ + OK = 0, + CORRUPTED, + NO_SHAPES +}; + +FeedStatus ReadFeed(gtfs::Feed & feed) +{ + // First we read shapes. If there are no shapes in feed we do not need to read all the required + // files - agencies, stops, etc. + if (auto res = feed.read_shapes(); res != gtfs::ResultCode::OK) + { + LOG(LWARNING, ("Could not get shapes.", res.message)); + return FeedStatus::NO_SHAPES; + } + + if (feed.get_shapes().empty()) + return FeedStatus::NO_SHAPES; + + // We try to parse required for json files and return error in case of invalid file content. + if (auto res = feed.read_agencies(); res != gtfs::ResultCode::OK) + { + LOG(LWARNING, ("Could not parse agencies.", res.message)); + return FeedStatus::CORRUPTED; + } + + if (auto res = feed.read_routes(); res != gtfs::ResultCode::OK) + { + LOG(LWARNING, ("Could not parse routes.", res.message)); + return FeedStatus::CORRUPTED; + } + + if (auto res = feed.read_trips(); res != gtfs::ResultCode::OK) + { + LOG(LWARNING, ("Could not parse trips.", res.message)); + return FeedStatus::CORRUPTED; + } + + if (auto res = feed.read_stops(); res != gtfs::ResultCode::OK) + { + LOG(LWARNING, ("Could not parse stops.", res.message)); + return FeedStatus::CORRUPTED; + } + + if (auto res = feed.read_stop_times(); res != gtfs::ResultCode::OK) + { + LOG(LWARNING, ("Could not parse stop times.", res.message)); + return FeedStatus::CORRUPTED; + } + + // We try to parse optional for json files and do not return error in case of invalid file + // content, only log warning message. + if (auto res = feed.read_calendar(); gtfs::ErrorParsingOptionalFile(res)) + LOG(LINFO, ("Could not parse calendar.", res.message)); + + if (auto res = feed.read_calendar_dates(); gtfs::ErrorParsingOptionalFile(res)) + LOG(LINFO, ("Could not parse calendar dates.", res.message)); + + if (auto res = feed.read_frequencies(); gtfs::ErrorParsingOptionalFile(res)) + LOG(LINFO, ("Could not parse frequencies.", res.message)); + + if (auto res = feed.read_transfers(); gtfs::ErrorParsingOptionalFile(res)) + LOG(LINFO, ("Could not parse transfers.", res.message)); + + if (feed.read_feed_info() == gtfs::ResultCode::OK) + LOG(LINFO, ("Feed info is present.")); + + return FeedStatus::OK; +} + +int main(int argc, char ** argv) +{ + google::SetUsageMessage("Reads GTFS feeds, produces json with global ids for generator."); + google::ParseCommandLineFlags(&argc, &argv, true); + auto const toolName = base::GetNameFromFullPath(argv[0]); + + if (FLAGS_path_mapping.empty() || FLAGS_path_gtfs_feeds.empty() || FLAGS_path_json.empty()) + { + LOG(LWARNING, ("Some of the required options are not present.")); + google::ShowUsageWithFlagsRestrict(argv[0], toolName.c_str()); + return -1; + } + + if (!Platform::IsDirectory(FLAGS_path_gtfs_feeds) || !Platform::IsDirectory(FLAGS_path_json) || + !Platform::IsDirectory(FLAGS_path_resources)) + { + LOG(LWARNING, + ("Some paths set in options are not valid. Check the directories:", + FLAGS_path_gtfs_feeds, FLAGS_path_json, FLAGS_path_resources)); + google::ShowUsageWithFlagsRestrict(argv[0], toolName.c_str()); + return -1; + } + + auto const gtfsFeeds = GetGtfsFeedsInDirectory(FLAGS_path_gtfs_feeds); + + if (gtfsFeeds.empty()) + { + LOG(LERROR, ("No subdirectories with GTFS feeds found in", FLAGS_path_gtfs_feeds)); + return -1; + } + + std::vector invalidFeeds; + + size_t feedsWithNoShapesCount = 0; + size_t feedsNotDumpedCount = 0; + size_t feedsDumped = 0; + size_t feedsTotal = gtfsFeeds.size(); + bool pass = true; + + transit::IdGenerator generator(FLAGS_path_mapping); + + GetPlatform().SetResourceDir(FLAGS_path_resources); + transit::ColorPicker colorPicker; + + for (size_t i = 0; i < gtfsFeeds.size(); ++i) + { + base::Timer feedTimer; + auto feedPath = gtfsFeeds[i]; + + if (SkipFeed(feedPath, pass)) + { + ++feedsTotal; + LOG(LINFO, ("Skipped", feedPath)); + continue; + } + + bool stop = StopOnFeed(feedPath); + if (stop) + feedsTotal -= (gtfsFeeds.size() - i - 1); + + ExtendPath(feedPath); + LOG(LINFO, ("Handling feed", feedPath)); + + gtfs::Feed feed(feedPath); + + if (auto const res = ReadFeed(feed); res != FeedStatus::OK) + { + if (res == FeedStatus::NO_SHAPES) + feedsWithNoShapesCount++; + else + invalidFeeds.push_back(feedPath); + + if (stop) + break; + continue; + } + + transit::WorldFeed globalFeed(generator, colorPicker); + + if (!globalFeed.SetFeed(std::move(feed))) + { + LOG(LINFO, ("Error transforming feed for json representation.")); + ++feedsNotDumpedCount; + if (stop) + break; + continue; + } + + bool const saved = globalFeed.Save(FLAGS_path_json, i == 0 /* overwrite */); + if (saved) + ++feedsDumped; + else + ++feedsNotDumpedCount; + + LOG(LINFO, ("Merged:", saved ? "yes" : "no", "time", feedTimer.ElapsedSeconds(), "s")); + + if (stop) + break; + } + + generator.Save(); + + LOG(LINFO, ("Corrupted feeds paths:", invalidFeeds)); + LOG(LINFO, ("Corrupted feeds:", invalidFeeds.size(), "/", feedsTotal)); + LOG(LINFO, ("Feeds with no shapes:", feedsWithNoShapesCount, "/", feedsTotal)); + LOG(LINFO, ("Feeds parsed but not dumped:", feedsNotDumpedCount, "/", feedsTotal)); + LOG(LINFO, ("Total dumped feeds:", feedsDumped, "/", feedsTotal)); + LOG(LINFO, ("Bad stop sequences:", transit::WorldFeed::GetCorruptedStopSequenceCount())); + return 0; +} diff --git a/transit/world_feed/world_feed.cpp b/transit/world_feed/world_feed.cpp new file mode 100644 index 00000000000..f41bc9b554b --- /dev/null +++ b/transit/world_feed/world_feed.cpp @@ -0,0 +1,1345 @@ +#include "transit/world_feed/world_feed.hpp" + +#include "routing/fake_feature_ids.hpp" + +#include "transit/world_feed/date_time_helpers.hpp" +#include "transit/world_feed/feed_helpers.hpp" + +#include "indexer/fake_feature_ids.hpp" + +#include "platform/platform.hpp" + +#include "coding/string_utf8_multilang.hpp" + +#include "base/assert.hpp" +#include "base/file_name_utils.hpp" +#include "base/logging.hpp" +#include "base/newtype.hpp" + +#include +#include +#include +#include +#include +#include + +#include "3party/boost/boost/algorithm/string.hpp" +#include "3party/boost/boost/container_hash/hash.hpp" + +#include "3party/jansson/myjansson.hpp" + +namespace +{ +template +auto BuildHash(Values... values) +{ + static std::string const delimiter = "_"; + + size_t constexpr paramsCount = sizeof...(Values); + size_t const delimitersSize = (paramsCount - 1) * delimiter.size(); + size_t const totalSize = (delimitersSize + ... + values.size()); + + std::string hash; + hash.reserve(totalSize); + (hash.append(values + delimiter), ...); + hash.pop_back(); + + return hash; +} + +void WriteJson(json_t * node, std::ofstream & output) +{ + std::unique_ptr buffer(json_dumps(node, JSON_COMPACT)); + std::string record(buffer.get()); + output << record << std::endl; +} + +base::JSONPtr PointToJson(m2::PointD const & point) +{ + auto coords = base::NewJSONObject(); + ToJSONObject(*coords, "x", point.x); + ToJSONObject(*coords, "y", point.y); + return coords; +} + +base::JSONPtr ShapeLinkToJson(transit::ShapeLink const & shapeLink) +{ + auto node = base::NewJSONObject(); + ToJSONObject(*node, "id", shapeLink.m_shapeId); + ToJSONObject(*node, "start_index", shapeLink.m_startIndex); + ToJSONObject(*node, "end_index", shapeLink.m_endIndex); + return node; +} + +base::JSONPtr StopIdsToJson(transit::IdList const & stopIds) +{ + auto idArr = base::NewJSONArray(); + + for (auto const & stopId : stopIds) + { + auto nodeId = base::NewJSONInt(stopId); + json_array_append_new(idArr.get(), nodeId.release()); + } + + return idArr; +} + +base::JSONPtr TranslationsToJson(transit::Translations const & translations) +{ + auto translationsArr = base::NewJSONArray(); + + for (auto const & [lang, text] : translations) + { + auto translationJson = base::NewJSONObject(); + ToJSONObject(*translationJson, "lang", lang); + ToJSONObject(*translationJson, "text", text); + json_array_append_new(translationsArr.get(), translationJson.release()); + } + + return translationsArr; +} + +template +bool DumpData(T const & container, std::string const & path, bool overwrite) +{ + std::ofstream output; + output.exceptions(std::ofstream::failbit | std::ofstream::badbit); + + try + { + std::ios_base::openmode mode = overwrite ? std::ofstream::trunc : std::ofstream::app; + output.open(path, mode); + + if (!output.is_open()) + return false; + + container.Write(output); + } + catch (std::ofstream::failure const & se) + { + LOG(LWARNING, ("Exception saving json to file", path, se.what())); + return false; + } + + return true; +} + +struct StopOnShape +{ + transit::TransitId m_id = 0; + size_t m_index = 0; +}; + +size_t GetStopIndex(std::unordered_map> const & stopIndexes, + transit::TransitId id, size_t index = 0) +{ + auto it = stopIndexes.find(id); + CHECK(it != stopIndexes.end(), (id)); + CHECK(index < it->second.size(), (index, it->second.size())); + + return *(it->second.begin() + index); +} + +std::pair GetStopPairOnShape( + std::unordered_map> const & stopIndexes, + transit::StopsOnLines const & stopsOnLines, size_t index) +{ + auto const & stopIds = stopsOnLines.m_stopSeq; + + CHECK(!stopIds.empty(), ()); + CHECK_LESS(index, stopIds.size() - 1, ()); + + StopOnShape stop1; + StopOnShape stop2; + + stop1.m_id = stopIds[index]; + stop2.m_id = (!stopsOnLines.m_isValid && stopIds.size() == 1) ? stop1.m_id : stopIds[index + 1]; + + if (stopsOnLines.m_isValid) + { + stop1.m_index = GetStopIndex(stopIndexes, stop1.m_id); + stop2.m_index = stop1.m_id == stop2.m_id ? GetStopIndex(stopIndexes, stop1.m_id, 1) + : GetStopIndex(stopIndexes, stop2.m_id); + } + return {stop1, stop2}; +} + +struct Link +{ + Link(transit::TransitId lineId, transit::TransitId shapeId, size_t shapeSize); + transit::TransitId m_lineId; + transit::TransitId m_shapeId; + size_t m_shapeSize; +}; + +Link::Link(transit::TransitId lineId, transit::TransitId shapeId, size_t shapeSize) + : m_lineId(lineId), m_shapeId(shapeId), m_shapeSize(shapeSize) +{ +} +} // namespace + +namespace transit +{ +// Static fields. +std::unordered_set WorldFeed::m_agencyHashes; +size_t WorldFeed::m_badStopSeqCount = 0; + +EdgeId::EdgeId(TransitId fromStopId, TransitId toStopId, TransitId lineId) + : m_fromStopId(fromStopId), m_toStopId(toStopId), m_lineId(lineId) +{ +} + +bool EdgeId::operator==(EdgeId const & other) const +{ + return std::tie(m_fromStopId, m_toStopId, m_lineId) == + std::tie(other.m_fromStopId, other.m_toStopId, other.m_lineId); +} + +size_t EdgeIdHasher::operator()(EdgeId const & key) const +{ + size_t seed = 0; + boost::hash_combine(seed, key.m_fromStopId); + boost::hash_combine(seed, key.m_toStopId); + boost::hash_combine(seed, key.m_lineId); + return seed; +} + +bool operator<(EdgeTransferData const & d1, EdgeTransferData const & d2) +{ + return std::tie(d1.m_fromStopId, d1.m_toStopId) < std::tie(d2.m_fromStopId, d2.m_toStopId); +} + +ShapeData::ShapeData(std::vector const & points) : m_points(points) {} + +IdGenerator::IdGenerator(std::string const & idMappingPath) + : m_curId(routing::FakeFeatureIds::kTransitGraphFeaturesStart), m_idMappingPath(idMappingPath) +{ + LOG(LINFO, ("Inited generator with", m_curId, "start id and path to mappings", m_idMappingPath)); + CHECK(!m_idMappingPath.empty(), ()); + + if (!Platform::IsFileExistsByFullPath(m_idMappingPath)) + { + LOG(LINFO, ("Mapping", m_idMappingPath, "is not yet created.")); + return; + } + + std::ifstream mappingFile; + mappingFile.exceptions(std::ifstream::failbit | std::ifstream::badbit); + + try + { + mappingFile.open(m_idMappingPath); + CHECK(mappingFile.is_open(), ("Could not open", m_idMappingPath)); + + std::string idStr; + std::string hash; + bool inserted = false; + + if (!std::getline(mappingFile, idStr)) + { + LOG(LINFO, ("Mapping", m_idMappingPath, "is empty.")); + return; + } + + // The first line of the mapping file is current free id. + m_curId = static_cast(std::stol(idStr)); + CHECK(routing::FakeFeatureIds::IsTransitFeature(m_curId), (m_curId)); + + // Next lines are sequences of id and hash pairs, each on new line. + while (std::getline(mappingFile, idStr)) + { + std::getline(mappingFile, hash); + + auto const id = static_cast(std::stol(idStr)); + + std::tie(std::ignore, inserted) = m_hashToId.emplace(hash, id); + CHECK(inserted, ("Not unique", id, hash)); + CHECK(routing::FakeFeatureIds::IsTransitFeature(id), (id)); + } + } + catch (std::ifstream::failure const & se) + { + LOG(LERROR, ("Exception reading file with mappings", m_idMappingPath, se.what())); + } + + LOG(LINFO, ("Loaded", m_hashToId.size(), "hash-to-id mappings. Current free id:", m_curId)); +} + +TransitId IdGenerator::MakeId(const std::string & hash) +{ + CHECK(!hash.empty(), ("Empty hash cannot be added to the mapping.")); + + auto [it, inserted] = m_hashToId.emplace(hash, 0); + if (!inserted) + return it->second; + + it->second = m_curId++; + + return it->second; +} + +void IdGenerator::Save() +{ + LOG(LINFO, ("Started saving", m_hashToId.size(), "mappings to", m_idMappingPath)); + + std::ofstream mappingFile; + mappingFile.exceptions(std::ofstream::failbit | std::ofstream::badbit); + + try + { + mappingFile.open(m_idMappingPath, std::ofstream::out | std::ofstream::trunc); + CHECK(mappingFile.is_open(), ("Path to the mapping file does not exist:", m_idMappingPath)); + + mappingFile << m_curId << std::endl; + CHECK(routing::FakeFeatureIds::IsTransitFeature(m_curId), (m_curId)); + + for (auto const & [hash, id] : m_hashToId) + { + mappingFile << id << std::endl; + CHECK(routing::FakeFeatureIds::IsTransitFeature(id), (id)); + mappingFile << hash << std::endl; + } + } + catch (std::ofstream::failure const & se) + { + LOG(LERROR, ("Exception writing file with mappings", m_idMappingPath, se.what())); + } +} + +StopsOnLines::StopsOnLines(IdList const & ids) : m_stopSeq(ids) {} + +void StopData::UpdateTimetable(TransitId lineId, gtfs::StopTime const & stopTime) +{ + bool const arrivalIsSet = stopTime.arrival_time.is_provided(); + bool const departureIsSet = stopTime.departure_time.is_provided(); + + if (!arrivalIsSet && !departureIsSet) + return; + + auto arrival = arrivalIsSet ? stopTime.arrival_time : stopTime.departure_time; + auto departure = departureIsSet ? stopTime.departure_time : stopTime.arrival_time; + + arrival.limit_hours_to_24max(); + departure.limit_hours_to_24max(); + + auto const newRuleSeq = GetRuleSequenceOsmoh(arrival, departure); + auto [it, inserted] = m_timetable.emplace(lineId, osmoh::OpeningHours()); + if (inserted) + { + it->second = osmoh::OpeningHours({newRuleSeq}); + return; + } + + auto ruleSeq = it->second.GetRule(); + ruleSeq.push_back(newRuleSeq); + it->second = osmoh::OpeningHours({ruleSeq}); +} + +WorldFeed::WorldFeed(IdGenerator & generator, ColorPicker & colorPicker) + : m_idGenerator(generator), m_colorPicker(colorPicker) +{ +} + +bool WorldFeed::FillNetworks() +{ + bool inserted = false; + + for (const auto & agency : m_feed.get_agencies()) + { + // For one agency_name there can be multiple agency_id in the same feed. + std::string const agencyHash = BuildHash(agency.agency_id, agency.agency_name); + static size_t constexpr kMaxGtfsHashSize = 100; + if (m_gtfsHash.size() + agencyHash.size() <= kMaxGtfsHashSize) + m_gtfsHash += agencyHash; + + std::tie(std::ignore, inserted) = + m_gtfsIdToHash[FieldIdx::AgencyIdx].emplace(agency.agency_id, agencyHash); + + if (!inserted) + { + LOG(LINFO, ("agency_id duplicates in same feed:", agencyHash)); + continue; + } + + // agency_id is required when the dataset provides data for routes from more than one agency. + // Otherwise agency_id field can be empty in routes.txt and other files. So we add it too: + if (m_feed.get_agencies().size() == 1) + m_gtfsIdToHash[FieldIdx::AgencyIdx].emplace("", agencyHash); + + std::tie(std::ignore, inserted) = m_agencyHashes.insert(agencyHash); + if (!inserted) + { + LOG(LINFO, ("Agency hash copy from other feed:", agencyHash, "Skipped.")); + m_agencySkipList.insert(agencyHash); + } + + Translations translation; + translation[m_feedLanguage] = agency.agency_name; + + std::tie(std::ignore, inserted) = + m_networks.m_data.emplace(m_idGenerator.MakeId(agencyHash), translation); + CHECK(inserted, ()); + } + + return !m_networks.m_data.empty(); +} + +bool WorldFeed::FillRoutes() +{ + bool inserted = false; + + for (const auto & route : m_feed.get_routes()) + { + // Filters irrelevant types, e.g. taxi, subway. + if (!IsRelevantType(route.route_type)) + continue; + + std::string const routeType = ToString(route.route_type); + // Filters unrecognized types. + if (routeType.empty()) + continue; + + auto const itAgencyHash = m_gtfsIdToHash[FieldIdx::AgencyIdx].find(route.agency_id); + CHECK(itAgencyHash != m_gtfsIdToHash[FieldIdx::AgencyIdx].end(), (route.agency_id)); + + auto const & agencyHash = itAgencyHash->second; + CHECK(!agencyHash.empty(), ("Empty hash for agency id:", route.agency_id)); + + // Avoids duplicates of agencies with linked routes. + if (m_agencySkipList.find(agencyHash) != m_agencySkipList.end()) + continue; + + std::string const routeHash = BuildHash(agencyHash, route.route_id); + std::tie(std::ignore, inserted) = + m_gtfsIdToHash[FieldIdx::RoutesIdx].emplace(route.route_id, routeHash); + CHECK(inserted, (route.route_id, routeHash)); + + RouteData data; + data.m_networkId = m_idGenerator.MakeId(agencyHash); + data.m_color = m_colorPicker.GetNearestColor(route.route_color); + data.m_routeType = routeType; + data.m_title[m_feedLanguage] = + route.route_long_name.empty() ? route.route_short_name : route.route_long_name; + + std::tie(std::ignore, inserted) = + m_routes.m_data.emplace(m_idGenerator.MakeId(routeHash), data); + CHECK(inserted, ()); + } + + return !m_routes.m_data.empty(); +} + +bool WorldFeed::SetFeedLanguage() +{ + static std::string const kNativeForCountry = "default"; + + m_feedLanguage = m_feed.get_feed_info().feed_lang; + if (m_feedLanguage.empty()) + { + m_feedLanguage = kNativeForCountry; + return false; + } + + boost::algorithm::to_lower(m_feedLanguage); + + StringUtf8Multilang multilang; + if (multilang.GetLangIndex(m_feedLanguage) != StringUtf8Multilang::kUnsupportedLanguageCode) + return true; + + LOG(LINFO, ("Unsupported language:", m_feedLanguage)); + m_feedLanguage = kNativeForCountry; + return false; +} + +bool WorldFeed::AddShape(GtfsIdToHash::iterator & iter, std::string const & gtfsShapeId, + TransitId lineId) +{ + auto const & shapeItems = m_feed.get_shape(gtfsShapeId, true); + if (shapeItems.size() < 2) + { + LOG(LINFO, ("Invalid shape. Length:", shapeItems.size(), "Shape id", gtfsShapeId)); + return false; + } + + std::string const shapeHash = BuildHash(m_gtfsHash, shapeItems[0].shape_id); + iter->second = shapeHash; + auto const shapeId = m_idGenerator.MakeId(shapeHash); + + auto [it, inserted] = m_shapes.m_data.emplace(shapeId, ShapeData()); + + if (inserted) + { + std::vector points; + // Reserve items also for future insertion of stops projections: + points.reserve(shapeItems.size() * 1.3); + + for (auto const & point : shapeItems) + points.push_back(mercator::FromLatLon(point.shape_pt_lat, point.shape_pt_lon)); + + it->second.m_points = points; + } + + it->second.m_lineIds.insert(lineId); + + return true; +} + +bool WorldFeed::UpdateStop(TransitId stopId, gtfs::StopTime const & stopTime, + std::string const & stopHash, TransitId lineId) +{ + auto [it, inserted] = m_stops.m_data.emplace(stopId, StopData()); + + // If the stop is already present we replenish the schedule and return. + if (!inserted) + { + it->second.UpdateTimetable(lineId, stopTime); + return true; + } + + auto const stop = m_feed.get_stop(stopTime.stop_id); + if (stop) + { + m_gtfsIdToHash[FieldIdx::StopsIdx].emplace(stopTime.stop_id, stopHash); + } + else + { + LOG(LINFO, + ("stop is present in stop_times, but not in stops:", stopId, stopTime.stop_id, lineId)); + return false; + } + + StopData data; + data.m_point = mercator::FromLatLon(stop->stop_lat, stop->stop_lon); + data.m_title[m_feedLanguage] = stop->stop_name; + data.m_gtfsParentId = stop->parent_station; + data.UpdateTimetable(lineId, stopTime); + it->second = data; + return true; +} + +std::pair WorldFeed::GetStopIdAndHash(std::string const & stopGtfsId) +{ + std::string const stopHash = BuildHash(m_gtfsHash, stopGtfsId); + auto const stopId = m_idGenerator.MakeId(stopHash); + return {stopId, stopHash}; +} + +bool WorldFeed::FillStopsEdges() +{ + gtfs::StopTimes allStopTimes = m_feed.get_stop_times(); + std::sort( + allStopTimes.begin(), allStopTimes.end(), + [](const gtfs::StopTime & t1, const gtfs::StopTime & t2) { return t1.trip_id < t2.trip_id; }); + + std::vector::iterator> linesForRemoval; + + for (auto it = m_lines.m_data.begin(); it != m_lines.m_data.end(); ++it) + { + auto const & lineId = it->first; + LineData & lineData = it->second; + TransitId const & shapeId = lineData.m_shapeLink.m_shapeId; + gtfs::StopTimes stopTimes = GetStopTimesForTrip(allStopTimes, lineData.m_gtfsTripId); + + if (stopTimes.size() < 2) + { + LOG(LINFO, ("Invalid stop times count for trip:", stopTimes.size(), lineData.m_gtfsTripId)); + linesForRemoval.push_back(it); + continue; + } + + lineData.m_stopIds.reserve(stopTimes.size()); + + for (size_t i = 0; i < stopTimes.size() - 1; ++i) + { + auto const & stopTime1 = stopTimes[i]; + auto const & stopTime2 = stopTimes[i + 1]; + + // This situation occurs even in valid GTFS feeds. + if (stopTime1.stop_id == stopTime2.stop_id && stopTimes.size() > 2) + continue; + + auto [stop1Id, stop1Hash] = GetStopIdAndHash(stopTime1.stop_id); + auto [stop2Id, stop2Hash] = GetStopIdAndHash(stopTime2.stop_id); + + lineData.m_stopIds.push_back(stop1Id); + if (!UpdateStop(stop1Id, stopTime1, stop1Hash, lineId)) + return false; + + if (i == stopTimes.size() - 2) + { + lineData.m_stopIds.push_back(stop2Id); + if (!UpdateStop(stop2Id, stopTime2, stop2Hash, lineId)) + return false; + } + + EdgeData data; + data.m_shapeLink.m_shapeId = shapeId; + data.m_weight = + stopTime2.arrival_time.get_total_seconds() - stopTime1.departure_time.get_total_seconds(); + + auto [itEdge, insertedEdge] = m_edges.m_data.emplace(EdgeId(stop1Id, stop2Id, lineId), data); + + // There can be two identical pairs of stops on the same trip. + if (!insertedEdge) + { + CHECK_EQUAL(itEdge->second.m_shapeLink.m_shapeId, data.m_shapeLink.m_shapeId, + (stopTime1.stop_id, stopTime2.stop_id)); + + // We choose the most pessimistic alternative. + if (data.m_weight > itEdge->second.m_weight) + itEdge->second = data; + } + } + } + + for (auto & it : linesForRemoval) + m_lines.m_data.erase(it); + + return !m_edges.m_data.empty(); +} + +bool WorldFeed::FillLinesAndShapes() +{ + for (const auto & trip : m_feed.get_trips()) + { + // We skip routes filtered on the route preparation stage. + auto const itRoute = m_gtfsIdToHash[FieldIdx::RoutesIdx].find(trip.route_id); + if (itRoute == m_gtfsIdToHash[FieldIdx::RoutesIdx].end()) + continue; + + // Skip trips with corrupted shapes. + if (trip.shape_id.empty()) + continue; + + std::string const & routeHash = itRoute->second; + std::string const lineHash = BuildHash(routeHash, trip.trip_id); + auto const lineId = m_idGenerator.MakeId(lineHash); + + auto [itShape, insertedShape] = m_gtfsIdToHash[ShapesIdx].emplace(trip.shape_id, ""); + + // Skip invalid shape. + if (!insertedShape && itShape->second.empty()) + continue; + + if (insertedShape) + { + // Skip trips with corrupted shapes. + if (!AddShape(itShape, trip.shape_id, lineId)) + continue; + } + + auto [it, inserted] = m_lines.m_data.emplace(lineId, LineData()); + if (!inserted) + { + LOG(LINFO, ("Duplicate trip_id:", trip.trip_id, m_gtfsHash)); + return false; + } + + TransitId const shapeId = m_idGenerator.MakeId(itShape->second); + + LineData data; + data.m_title[m_feedLanguage] = trip.trip_short_name; + data.m_routeId = m_idGenerator.MakeId(routeHash); + data.m_shapeId = shapeId; + data.m_gtfsTripId = trip.trip_id; + data.m_gtfsServiceId = trip.service_id; + // data.m_intervals and data.m_schedule will be filled on the next steps. + it->second = data; + + m_gtfsIdToHash[TripsIdx].emplace(trip.trip_id, lineHash); + } + + return !m_lines.m_data.empty() && !m_shapes.m_data.empty(); +} + +void WorldFeed::ModifyLinesAndShapes() +{ + std::vector links; + links.reserve(m_lines.m_data.size()); + + for (auto const & [lineId, lineData] : m_lines.m_data) + { + links.emplace_back(lineId, lineData.m_shapeId, + m_shapes.m_data[lineData.m_shapeId].m_points.size()); + } + + // We sort links by shape length so we could search for shapes in which i-th shape is included + // only in the left part of the array [0, i). + std::sort(links.begin(), links.end(), [](Link const & link1, Link const & link2) { + return link1.m_shapeSize > link2.m_shapeSize; + }); + + size_t subShapesCount = 0; + + // Shape ids of shapes fully contained in other shapes. + std::unordered_set shapesForRemoval; + + // Shape id matching to the line id linked to this shape id. + std::unordered_map matchingCache; + + for (size_t i = 1; i < links.size(); ++i) + { + auto const lineId = links[i].m_lineId; + auto & lineData = m_lines.m_data[lineId]; + auto const shapeId = links[i].m_shapeId; + + auto [itCache, inserted] = matchingCache.emplace(shapeId, 0); + if (!inserted) + { + if (itCache->second != 0) + { + lineData.m_shapeId = 0; + lineData.m_shapeLink = m_lines.m_data[itCache->second].m_shapeLink; + } + continue; + } + + auto const & points = m_shapes.m_data[shapeId].m_points; + + for (size_t j = 0; j < i; ++j) + { + auto const & curLineId = links[j].m_lineId; + + // We skip shapes which are already included to other shapes. + if (m_lines.m_data[curLineId].m_shapeId == 0) + continue; + + auto const curShapeId = links[j].m_shapeId; + + if (curShapeId == shapeId) + continue; + + auto const & curPoints = m_shapes.m_data[curShapeId].m_points; + + auto const it = std::search(curPoints.begin(), curPoints.end(), points.begin(), points.end()); + + if (it == curPoints.end()) + continue; + + // Shape with |points| polyline is fully contained in the shape with |curPoints| polyline. + lineData.m_shapeId = 0; + lineData.m_shapeLink.m_shapeId = curShapeId; + lineData.m_shapeLink.m_startIndex = std::distance(curPoints.begin(), it); + lineData.m_shapeLink.m_endIndex = lineData.m_shapeLink.m_startIndex + points.size() - 1; + itCache->second = lineId; + shapesForRemoval.insert(shapeId); + ++subShapesCount; + break; + } + } + + for (auto & [lineId, lineData] : m_lines.m_data) + { + if (lineData.m_shapeId == 0) + continue; + + lineData.m_shapeLink.m_shapeId = lineData.m_shapeId; + lineData.m_shapeLink.m_startIndex = 0; + lineData.m_shapeLink.m_endIndex = m_shapes.m_data[lineData.m_shapeId].m_points.size(); + + lineData.m_shapeId = 0; + } + + for (auto const shapeId : shapesForRemoval) + m_shapes.m_data.erase(shapeId); + + LOG(LINFO, ("Deleted", subShapesCount, "sub-shapes.", m_shapes.m_data.size(), "left.")); +} + +void WorldFeed::GetCalendarDates(osmoh::TRuleSequences & rules, CalendarCache & cache, + std::string const & serviceId) +{ + auto [it, inserted] = cache.emplace(serviceId, osmoh::TRuleSequences()); + if (inserted) + { + if (auto serviceDays = m_feed.get_calendar(serviceId); serviceDays) + { + GetServiceDaysOsmoh(serviceDays.value(), it->second); + MergeRules(rules, it->second); + } + } + else + { + MergeRules(rules, it->second); + } +} + +void WorldFeed::GetCalendarDatesExceptions(osmoh::TRuleSequences & rules, CalendarCache & cache, + std::string const & serviceId) +{ + auto [it, inserted] = cache.emplace(serviceId, osmoh::TRuleSequences()); + if (inserted) + { + auto exceptionDates = m_feed.get_calendar_dates(serviceId); + GetServiceDaysExceptionsOsmoh(exceptionDates, it->second); + } + + MergeRules(rules, it->second); +} + +LineIntervals WorldFeed::GetFrequencies(std::unordered_map & cache, + std::string const & tripId) +{ + auto [it, inserted] = cache.emplace(tripId, LineIntervals{}); + if (!inserted) + { + return it->second; + } + + auto const & frequencies = m_feed.get_frequencies(tripId); + if (frequencies.empty()) + return it->second; + + std::unordered_map intervals; + for (auto const & freq : frequencies) + { + osmoh::RuleSequence const seq = GetRuleSequenceOsmoh(freq.start_time, freq.end_time); + intervals[freq.headway_secs].push_back(seq); + } + + for (auto const & [headwayS, rules] : intervals) + { + LineInterval interval; + interval.m_headwayS = headwayS; + interval.m_timeIntervals = osmoh::OpeningHours(rules); + it->second.push_back(interval); + } + return it->second; +} + +bool WorldFeed::FillLinesSchedule() +{ + // Service id - to - rules mapping based on GTFS calendar. + CalendarCache cachedCalendar; + // Service id - to - rules mapping based on GTFS calendar dates. + CalendarCache cachedCalendarDates; + + // Trip id - to - headways for trips. + std::unordered_map cachedFrequencies; + + for (auto & [lineId, lineData] : m_lines.m_data) + { + osmoh::TRuleSequences rulesDates; + auto const & serviceId = lineData.m_gtfsServiceId; + + GetCalendarDates(rulesDates, cachedCalendar, serviceId); + GetCalendarDatesExceptions(rulesDates, cachedCalendarDates, serviceId); + + lineData.m_serviceDays = osmoh::OpeningHours(rulesDates); + + auto const & tripId = lineData.m_gtfsTripId; + lineData.m_intervals = GetFrequencies(cachedFrequencies, tripId); + } + + return !cachedCalendar.empty() || !cachedCalendarDates.empty(); +} + +bool WorldFeed::ProjectStopsToShape( + TransitId shapeId, std::vector & shape, IdList const & stopIds, + std::unordered_map> & stopsToIndexes) +{ + for (size_t i = 0; i < stopIds.size(); ++i) + { + auto const & stopId = stopIds[i]; + auto const itStop = m_stops.m_data.find(stopId); + CHECK(itStop != m_stops.m_data.end(), (stopId)); + auto const & stop = itStop->second; + + size_t const startIdx = i == 0 ? 0 : stopsToIndexes[stopIds[i - 1]].back(); + auto const [curIdx, pointInserted] = PrepareNearestPointOnTrack(stop.m_point, startIdx, shape); + + if (curIdx > shape.size()) + return false; + + if (pointInserted) + { + for (auto & indexesList : stopsToIndexes) + { + for (auto & stopIndex : indexesList.second) + { + if (stopIndex >= curIdx) + ++stopIndex; + } + } + + for (auto const & lineId : m_shapes.m_data[shapeId].m_lineIds) + { + auto & line = m_lines.m_data[lineId]; + + if (line.m_shapeLink.m_startIndex >= curIdx) + ++line.m_shapeLink.m_startIndex; + + if (line.m_shapeLink.m_endIndex >= curIdx) + ++line.m_shapeLink.m_endIndex; + } + } + + stopsToIndexes[stopId].push_back(curIdx); + } + + return true; +} + +std::unordered_map> WorldFeed::GetStopsForShapeMatching() +{ + // Shape id and list of stop sequences matched to the corresponding lines. + std::unordered_map> stopsOnShapes; + + // We build lists of stops relevant to corresponding shapes. There could be multiple different + // stops lists linked to the same shape. + // Example: line 8, shapeId 52, stops: [25, 26, 27]. line 9, shapeId 52, stops: [10, 9, 8, 7, 6]. + + for (auto const & [lineId, lineData] : m_lines.m_data) + { + auto & shapeData = stopsOnShapes[lineData.m_shapeLink.m_shapeId]; + bool found = false; + + for (auto & stopsOnLines : shapeData) + { + if (stopsOnLines.m_stopSeq == lineData.m_stopIds) + { + found = true; + stopsOnLines.m_lines.insert(lineId); + break; + } + } + + if (!found) + { + StopsOnLines stopsOnLines(lineData.m_stopIds); + stopsOnLines.m_lines.insert(lineId); + shapeData.emplace_back(stopsOnLines); + } + } + + return stopsOnShapes; +} + +size_t WorldFeed::ModifyShapes() +{ + auto stopsOnShapes = GetStopsForShapeMatching(); + size_t invalidStopSequences = 0; + + for (auto & [shapeId, stopsLists] : stopsOnShapes) + { + CHECK(!stopsLists.empty(), (shapeId)); + + auto it = m_shapes.m_data.find(shapeId); + CHECK(it != m_shapes.m_data.end(), (shapeId)); + auto & shape = it->second; + + std::unordered_map> stopToShapeIndex; + + for (auto & stopsOnLines : stopsLists) + { + if (stopsOnLines.m_stopSeq.size() < 2 || + !ProjectStopsToShape(shapeId, shape.m_points, stopsOnLines.m_stopSeq, stopToShapeIndex)) + { + stopsOnLines.m_isValid = false; + ++invalidStopSequences; + LOG(LINFO, + ("Error projecting stops to shape. trips count:", stopsOnLines.m_lines.size(), + "first trip GTFS id:", m_lines.m_data[*stopsOnLines.m_lines.begin()].m_gtfsTripId)); + } + } + + for (auto const & stopsOnLines : stopsLists) + { + IdList const & stopIds = stopsOnLines.m_stopSeq; + auto const & lineIds = stopsOnLines.m_lines; + auto indexes = stopToShapeIndex; + + for (size_t i = 0; i < stopIds.size() - 1; ++i) + { + auto const [stop1, stop2] = GetStopPairOnShape(indexes, stopsOnLines, i); + + for (auto const lineId : lineIds) + { + if (!stopsOnLines.m_isValid) + m_lines.m_data.erase(lineId); + + // Update |EdgeShapeLink| with shape segment start and end points. + auto itEdge = m_edges.m_data.find(EdgeId(stop1.m_id, stop2.m_id, lineId)); + if (itEdge == m_edges.m_data.end()) + continue; + + if (stopsOnLines.m_isValid) + { + itEdge->second.m_shapeLink.m_startIndex = stop1.m_index; + itEdge->second.m_shapeLink.m_endIndex = stop2.m_index; + } + else + { + m_edges.m_data.erase(itEdge); + } + } + + if (indexes[stop1.m_id].size() > 1) + indexes[stop1.m_id].erase(indexes[stop1.m_id].begin()); + } + } + } + + return invalidStopSequences; +} + +void WorldFeed::FillTransfers() +{ + bool inserted = false; + + for (auto const & transfer : m_feed.get_transfers()) + { + if (transfer.transfer_type == gtfs::TransferType::NotPossible) + continue; + + // Check that the two stops from the transfer are present in the global feed. + auto const itStop1 = m_gtfsIdToHash[FieldIdx::StopsIdx].find(transfer.from_stop_id); + if (itStop1 == m_gtfsIdToHash[FieldIdx::StopsIdx].end()) + continue; + + auto const itStop2 = m_gtfsIdToHash[FieldIdx::StopsIdx].find(transfer.to_stop_id); + if (itStop2 == m_gtfsIdToHash[FieldIdx::StopsIdx].end()) + continue; + + TransitId const & stop1Id = m_idGenerator.MakeId(itStop1->second); + TransitId const & stop2Id = m_idGenerator.MakeId(itStop2->second); + + // Usual case in GTFS feeds (don't ask why). We skip duplicates. + if (stop1Id == stop2Id) + continue; + + std::string const transitHash = BuildHash(itStop1->second, itStop2->second); + TransitId const transitId = m_idGenerator.MakeId(transitHash); + + TransferData data; + data.m_stopsIds = {stop1Id, stop2Id}; + data.m_point = m_stops.m_data.at(stop1Id).m_point; // TODO maybe change? + + std::tie(std::ignore, inserted) = m_transfers.m_data.emplace(transitId, data); + if (inserted) + { + EdgeTransferData edgeData; + edgeData.m_fromStopId = stop1Id; + edgeData.m_toStopId = stop2Id; + edgeData.m_weight = transfer.min_transfer_time; // Can be 0. + + std::tie(std::ignore, inserted) = m_edgesTransfers.m_data.insert(edgeData); + if (!inserted) + LOG(LINFO, ("Transfers copy", transfer.from_stop_id, transfer.to_stop_id)); + } + } +} + +void WorldFeed::FillGates() +{ + std::unordered_map> parentToGates; + for (auto const & stop : m_feed.get_stops()) + { + if (stop.location_type == gtfs::StopLocationType::EntranceExit && !stop.parent_station.empty()) + { + GateData gate; + gate.m_gtfsId = stop.stop_id; + gate.m_point = mercator::FromLatLon(stop.stop_lat, stop.stop_lon); + gate.m_isEntrance = true; + gate.m_isExit = true; + parentToGates[stop.parent_station].emplace_back(gate); + } + } + + if (parentToGates.empty()) + return; + + for (auto & [stopId, stopData] : m_stops.m_data) + { + if (stopData.m_gtfsParentId.empty()) + continue; + + auto it = parentToGates.find(stopData.m_gtfsParentId); + if (it == parentToGates.end()) + continue; + + for (auto & gate : it->second) + { + TimeFromGateToStop weight; + weight.m_stopId = stopId; + // We do not divide distance by average speed because average speed in stations, terminals, + // etc is roughly 1 meter/second. + weight.m_timeSeconds = mercator::DistanceOnEarth(stopData.m_point, gate.m_point); + + gate.m_weights.emplace_back(weight); + } + } + + for (auto const & pairParentGates : parentToGates) + { + auto const & gates = pairParentGates.second; + for (auto const & gate : gates) + { + TransitId id; + std::tie(id, std::ignore) = GetStopIdAndHash(gate.m_gtfsId); + m_gates.m_data.emplace(id, gate); + } + } +} + +bool WorldFeed::SetFeed(gtfs::Feed && feed) +{ + m_feed = std::move(feed); + m_gtfsIdToHash.resize(FieldIdx::IdxCount); + + // The order of the calls is important. First we set default feed language. Then fill networks. + // Then, based on network ids, we generate routes and so on. + + SetFeedLanguage(); + + if (!FillNetworks()) + { + LOG(LWARNING, ("Could not fill networks.")); + return false; + } + LOG(LINFO, ("Filled networks.")); + + if (!FillRoutes()) + { + LOG(LWARNING, ("Could not fill routes.")); + return false; + } + LOG(LINFO, ("Filled routes.")); + + if (!FillLinesAndShapes()) + { + LOG(LWARNING, ("Could not fill lines.", m_lines.m_data.size())); + return false; + } + LOG(LINFO, ("Filled lines and shapes.")); + + ModifyLinesAndShapes(); + LOG(LINFO, ("Modified lines and shapes.")); + + if (!FillLinesSchedule()) + { + LOG(LWARNING, ("Could not fill schedule for lines.")); + return false; + } + LOG(LINFO, ("Filled schedule for lines.")); + + if (!FillStopsEdges()) + { + LOG(LWARNING, ("Could not fill stops", m_stops.m_data.size())); + return false; + } + LOG(LINFO, ("Filled stop timetables and road graph edges.")); + + m_badStopSeqCount += ModifyShapes(); + LOG(LINFO, ("Modified shapes.")); + + FillTransfers(); + LOG(LINFO, ("Filled transfers.")); + + FillGates(); + LOG(LINFO, ("Filled gates.")); + return true; +} + +void Networks::Write(std::ofstream & stream) const +{ + for (auto const & [networkId, networkTitle] : m_data) + { + auto node = base::NewJSONObject(); + + ToJSONObject(*node, "id", networkId); + json_object_set_new(node.get(), "title", TranslationsToJson(networkTitle).release()); + + WriteJson(node.get(), stream); + } +} + +void Routes::Write(std::ofstream & stream) const +{ + for (auto const & [routeId, route] : m_data) + { + auto node = base::NewJSONObject(); + ToJSONObject(*node, "id", routeId); + ToJSONObject(*node, "network_id", route.m_networkId); + ToJSONObject(*node, "color", route.m_color); + ToJSONObject(*node, "type", route.m_routeType); + json_object_set_new(node.get(), "title", TranslationsToJson(route.m_title).release()); + + WriteJson(node.get(), stream); + } +} + +void Lines::Write(std::ofstream & stream) const +{ + for (auto const & [lineId, line] : m_data) + { + auto node = base::NewJSONObject(); + ToJSONObject(*node, "id", lineId); + ToJSONObject(*node, "route_id", line.m_routeId); + json_object_set_new(node.get(), "shape", ShapeLinkToJson(line.m_shapeLink).release()); + + json_object_set_new(node.get(), "title", TranslationsToJson(line.m_title).release()); + json_object_set_new(node.get(), "stops_ids", StopIdsToJson(line.m_stopIds).release()); + ToJSONObject(*node, "service_days", ToString(line.m_serviceDays)); + + auto intervalsArr = base::NewJSONArray(); + + for (auto const & [intervalS, openingHours] : line.m_intervals) + { + auto scheduleItem = base::NewJSONObject(); + ToJSONObject(*scheduleItem, "interval_s", intervalS); + ToJSONObject(*scheduleItem, "service_hours", ToString(openingHours)); + json_array_append_new(intervalsArr.get(), scheduleItem.release()); + } + + json_object_set_new(node.get(), "intervals", intervalsArr.release()); + + WriteJson(node.get(), stream); + } +} + +void Shapes::Write(std::ofstream & stream) const +{ + for (auto const & [shapeId, shape] : m_data) + { + auto node = base::NewJSONObject(); + ToJSONObject(*node, "id", shapeId); + auto pointsArr = base::NewJSONArray(); + + for (auto const & point : shape.m_points) + json_array_append_new(pointsArr.get(), PointToJson(point).release()); + + json_object_set_new(node.get(), "points", pointsArr.release()); + + WriteJson(node.get(), stream); + } +} + +void Stops::Write(std::ofstream & stream) const +{ + for (auto const & [stopId, stop] : m_data) + { + auto node = base::NewJSONObject(); + ToJSONObject(*node, "id", stopId); + + json_object_set_new(node.get(), "point", PointToJson(stop.m_point).release()); + json_object_set_new(node.get(), "title", TranslationsToJson(stop.m_title).release()); + + auto timeTableArr = base::NewJSONArray(); + + for (auto const & [lineId, schedule] : stop.m_timetable) + { + auto scheduleItem = base::NewJSONObject(); + ToJSONObject(*scheduleItem, "line_id", lineId); + ToJSONObject(*scheduleItem, "arrivals", ToString(schedule)); + json_array_append_new(timeTableArr.get(), scheduleItem.release()); + } + json_object_set_new(node.get(), "timetable", timeTableArr.release()); + + WriteJson(node.get(), stream); + } +} + +void Edges::Write(std::ofstream & stream) const +{ + for (auto const & [edgeId, edge] : m_data) + { + auto node = base::NewJSONObject(); + + ToJSONObject(*node, "line_id", edgeId.m_lineId); + ToJSONObject(*node, "stop_id_from", edgeId.m_fromStopId); + ToJSONObject(*node, "stop_id_to", edgeId.m_toStopId); + ToJSONObject(*node, "weight", edge.m_weight); + json_object_set_new(node.get(), "shape", ShapeLinkToJson(edge.m_shapeLink).release()); + + WriteJson(node.get(), stream); + } +} + +void EdgesTransfer::Write(std::ofstream & stream) const +{ + for (auto const & edge : m_data) + { + auto node = base::NewJSONObject(); + + ToJSONObject(*node, "stop_id_from", edge.m_fromStopId); + ToJSONObject(*node, "stop_id_to", edge.m_toStopId); + ToJSONObject(*node, "weight", edge.m_weight); + + WriteJson(node.get(), stream); + } +} + +void Transfers::Write(std::ofstream & stream) const +{ + for (auto const & [transferId, transfer] : m_data) + { + auto node = base::NewJSONObject(); + + ToJSONObject(*node, "id", transferId); + json_object_set_new(node.get(), "point", PointToJson(transfer.m_point).release()); + json_object_set_new(node.get(), "stops_ids", StopIdsToJson(transfer.m_stopsIds).release()); + + WriteJson(node.get(), stream); + } +} + +void Gates::Write(std::ofstream & stream) const +{ + for (auto const & [gateId, gate] : m_data) + { + if (gate.m_weights.empty()) + continue; + + auto node = base::NewJSONObject(); + ToJSONObject(*node, "id", gateId); + + auto weightsArr = base::NewJSONArray(); + + for (auto const & weight : gate.m_weights) + { + auto weightJson = base::NewJSONObject(); + ToJSONObject(*weightJson, "stop_id", weight.m_stopId); + ToJSONObject(*weightJson, "time_to_stop", weight.m_timeSeconds); + json_array_append_new(weightsArr.get(), weightJson.release()); + } + + json_object_set_new(node.get(), "weights", weightsArr.release()); + ToJSONObject(*node, "exit", gate.m_isExit); + ToJSONObject(*node, "entrance", gate.m_isEntrance); + + json_object_set_new(node.get(), "point", PointToJson(gate.m_point).release()); + + WriteJson(node.get(), stream); + } +} + +bool WorldFeed::Save(std::string const & worldFeedDir, bool overwrite) +{ + CHECK(!worldFeedDir.empty(), ()); + CHECK(!m_networks.m_data.empty(), ()); + + if (m_routes.m_data.empty() || m_lines.m_data.empty() || m_stops.m_data.empty() || + m_shapes.m_data.empty()) + { + LOG(LINFO, ("Skipping feed with routes, lines, stops, shapes sizes:", m_routes.m_data.size(), + m_lines.m_data.size(), m_stops.m_data.size(), m_shapes.m_data.size())); + return false; + } + + CHECK(!m_edges.m_data.empty(), ()); + LOG(LINFO, ("Saving feed to", worldFeedDir)); + CHECK(DumpData(m_networks, base::JoinPath(worldFeedDir, kNetworksFile), overwrite), ()); + CHECK(DumpData(m_routes, base::JoinPath(worldFeedDir, kRoutesFile), overwrite), ()); + CHECK(DumpData(m_lines, base::JoinPath(worldFeedDir, kLinesFile), overwrite), ()); + CHECK(DumpData(m_shapes, base::JoinPath(worldFeedDir, kShapesFile), overwrite), ()); + CHECK(DumpData(m_stops, base::JoinPath(worldFeedDir, kStopsFile), overwrite), ()); + CHECK(DumpData(m_edges, base::JoinPath(worldFeedDir, kEdgesFile), overwrite), ()); + CHECK(DumpData(m_edgesTransfers, base::JoinPath(worldFeedDir, kEdgesTransferFile), overwrite), + ()); + CHECK(DumpData(m_transfers, base::JoinPath(worldFeedDir, kTransfersFile), overwrite), ()); + CHECK(DumpData(m_gates, base::JoinPath(worldFeedDir, kGatesFile), overwrite), ()); + + return true; +} +} // namespace transit diff --git a/transit/world_feed/world_feed.hpp b/transit/world_feed/world_feed.hpp new file mode 100644 index 00000000000..eeb777fd975 --- /dev/null +++ b/transit/world_feed/world_feed.hpp @@ -0,0 +1,370 @@ +#pragma once +#include "transit/world_feed/color_picker.hpp" + +#include "geometry/mercator.hpp" +#include "geometry/point2d.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include "3party/just_gtfs/just_gtfs.h" +#include "3party/opening_hours/opening_hours.hpp" + +#include "defines.hpp" + +namespace transit +{ +// File names for saving resulting data exported from GTFS. +inline std::string const kTransitFileExtension = std::string(TRANSIT_FILE_EXTENSION); +inline std::string const kNetworksFile = "networks" + kTransitFileExtension; +inline std::string const kRoutesFile = "routes" + kTransitFileExtension; +inline std::string const kLinesFile = "lines" + kTransitFileExtension; +inline std::string const kShapesFile = "shapes" + kTransitFileExtension; +inline std::string const kStopsFile = "stops" + kTransitFileExtension; +inline std::string const kEdgesFile = "edges" + kTransitFileExtension; +inline std::string const kEdgesTransferFile = "edges_transfer" + kTransitFileExtension; +inline std::string const kTransfersFile = "transfers" + kTransitFileExtension; +inline std::string const kGatesFile = "gates" + kTransitFileExtension; + +// Unique id persistent between re-runs. Generated based on the unique string hash of the +// GTFS entity. Lies in the interval |routing::FakeFeatureIds::IsTransitFeature()|. +// If the GTFS entity is renamed or the new GTFS feed is added the new id is generated by +// |IdGenerator::MakeId()|. +using TransitId = uint32_t; + +// Generates globally unique TransitIds mapped to the GTFS entities hashes. +class IdGenerator +{ +public: + explicit IdGenerator(std::string const & idMappingPath); + void Save(); + + TransitId MakeId(std::string const & hash); + +private: + std::unordered_map m_hashToId; + TransitId m_curId; + std::string m_idMappingPath; +}; + +// Here are MAPS.ME representations for GTFS entities, e.g. networks for GTFS agencies. +// https://developers.google.com/transit/gtfs/reference + +// Mapping of language id to text. +using Translations = std::unordered_map; + +struct Networks +{ + void Write(std::ofstream & stream) const; + + // Id to agency name mapping. + std::unordered_map m_data; +}; + +struct RouteData +{ + TransitId m_networkId = 0; + std::string m_routeType; + Translations m_title; + std::string m_color; +}; + +struct Routes +{ + void Write(std::ofstream & stream) const; + + std::unordered_map m_data; +}; + +struct LineInterval +{ + size_t m_headwayS = 0; + osmoh::OpeningHours m_timeIntervals; +}; + +using LineIntervals = std::vector; +using IdList = std::vector; + +// Link to the shape: its id and indexes in the corresponding polyline. +struct ShapeLink +{ + TransitId m_shapeId = 0; + size_t m_startIndex = 0; + size_t m_endIndex = 0; +}; + +struct LineData +{ + TransitId m_routeId = 0; + ShapeLink m_shapeLink; + Translations m_title; + // Sequence of stops along the line from first to last. + IdList m_stopIds; + + // Transport intervals depending on the day timespans. + std::vector m_intervals; + // Monthdays and weekdays ranges on which the line is at service. + // Exceptions in service schedule. Explicitly activates or disables service by dates. + osmoh::OpeningHours m_serviceDays; + + // Fields not intended to be exported to json. + TransitId m_shapeId = 0; + std::string m_gtfsTripId; + std::string m_gtfsServiceId; +}; + +struct Lines +{ + void Write(std::ofstream & stream) const; + + std::unordered_map m_data; +}; + +struct ShapeData +{ + ShapeData() = default; + explicit ShapeData(std::vector const & points); + + std::vector m_points; + // Field not for dumping to json: + std::unordered_set m_lineIds; +}; + +struct Shapes +{ + void Write(std::ofstream & stream) const; + + std::unordered_map m_data; +}; + +struct StopData +{ + void UpdateTimetable(TransitId lineId, gtfs::StopTime const & stopTime); + + m2::PointD m_point; + Translations m_title; + // For each line id there is a schedule. + std::unordered_map m_timetable; + + // Field not intended for dumping to json: + std::string m_gtfsParentId; +}; + +struct Stops +{ + void Write(std::ofstream & stream) const; + + std::unordered_map m_data; +}; + +struct EdgeId +{ + EdgeId() = default; + EdgeId(TransitId fromStopId, TransitId toStopId, TransitId lineId); + + bool operator==(EdgeId const & other) const; + + TransitId m_fromStopId = 0; + TransitId m_toStopId = 0; + TransitId m_lineId = 0; +}; + +struct EdgeIdHasher +{ + size_t operator()(EdgeId const & key) const; +}; + +struct EdgeData +{ + ShapeLink m_shapeLink; + size_t m_weight = 0; +}; + +struct Edges +{ + void Write(std::ofstream & stream) const; + + std::unordered_map m_data; +}; + +struct EdgeTransferData +{ + TransitId m_fromStopId = 0; + TransitId m_toStopId = 0; + size_t m_weight = 0; +}; + +bool operator<(EdgeTransferData const & d1, EdgeTransferData const & d2); + +struct EdgesTransfer +{ + void Write(std::ofstream & stream) const; + + std::set m_data; +}; + +struct TransferData +{ + m2::PointD m_point; + IdList m_stopsIds; +}; + +struct Transfers +{ + void Write(std::ofstream & stream) const; + + std::unordered_map m_data; +}; + +struct TimeFromGateToStop +{ + TransitId m_stopId = 0; + size_t m_timeSeconds = 0; +}; + +struct GateData +{ + bool m_isEntrance = false; + bool m_isExit = false; + m2::PointD m_point; + std::vector m_weights; + + // Field not intended for dumping to json: + std::string m_gtfsId; +}; + +struct Gates +{ + void Write(std::ofstream & stream) const; + + std::unordered_map m_data; +}; + +// Indexes for WorldFeed |m_gtfsIdToHash| field. For each type of GTFS entity, e.g. agency or stop, +// there is distinct mapping located by its own |FieldIdx| index in the |m_gtfsIdToHash|. +enum FieldIdx +{ + AgencyIdx = 0, + StopsIdx, + RoutesIdx, + TripsIdx, + ShapesIdx, + IdxCount +}; + +using GtfsIdToHash = std::unordered_map; +using CalendarCache = std::unordered_map; + +struct StopsOnLines +{ + explicit StopsOnLines(IdList const & ids); + + IdList m_stopSeq; + std::unordered_set m_lines; + bool m_isValid = true; +}; + +// Class for merging scattered GTFS feeds into one World feed with static ids. +class WorldFeed +{ +public: + WorldFeed(IdGenerator & generator, ColorPicker & colorPicker); + // Transforms GTFS feed into the global feed. + bool SetFeed(gtfs::Feed && feed); + + // Dumps global feed to |world_feed_path|. + bool Save(std::string const & worldFeedDir, bool overwrite); + + inline static size_t GetCorruptedStopSequenceCount() { return m_badStopSeqCount; } + +private: + bool SetFeedLanguage(); + // Fills networks from GTFS agencies data. + bool FillNetworks(); + // Fills routes from GTFS foutes data. + bool FillRoutes(); + // Fills lines and corresponding shapes from GTFS trips and shapes. + bool FillLinesAndShapes(); + // Deletes shapes which are sub-shapes and refreshes corresponding links in lines. + void ModifyLinesAndShapes(); + // Gets service monthday open/closed ranges, weekdays and exceptions in schedule. + bool FillLinesSchedule(); + // Gets frequencies of trips from GTFS. + + // Adds shape with mercator points instead of WGS83 lat/lon. + bool AddShape(GtfsIdToHash::iterator & iter, std::string const & gtfsShapeId, TransitId lineId); + // Fills stops data and builds corresponding edges for the road graph. + bool FillStopsEdges(); + + // Generates globally unique id and hash for the stop by its |stopGtfsId|. + std::pair GetStopIdAndHash(std::string const & stopGtfsId); + + // Adds new stop with |stopId| and fills it with GTFS data by |gtfsId| or just + // links to it |lineId|. + bool UpdateStop(TransitId stopId, gtfs::StopTime const & stopTime, std::string const & stopHash, + TransitId lineId); + + std::unordered_map> GetStopsForShapeMatching(); + + // Adds stops projections to shapes. Updates corresponding links to shapes. + size_t ModifyShapes(); + // Fills transfers based on GTFS transfers. + void FillTransfers(); + // Fills gates based on GTFS stops. + void FillGates(); + + bool ProjectStopsToShape(TransitId shapeId, std::vector & shape, + IdList const & stopIds, + std::unordered_map> & stopsToIndexes); + + // Extracts data from GTFS calendar for lines. + void GetCalendarDates(osmoh::TRuleSequences & rules, CalendarCache & cache, + std::string const & serviceId); + // Extracts data from GTFS calendar dates for lines. + void GetCalendarDatesExceptions(osmoh::TRuleSequences & rules, CalendarCache & cache, + std::string const & serviceId); + + LineIntervals GetFrequencies(std::unordered_map & cache, + std::string const & tripId); + // Current GTFS feed which is being merged to the global feed. + gtfs::Feed m_feed; + + // Entities for json'izing and feeding to the generator_tool. + Networks m_networks; + Routes m_routes; + Lines m_lines; + Shapes m_shapes; + Stops m_stops; + Edges m_edges; + EdgesTransfer m_edgesTransfers; + Transfers m_transfers; + Gates m_gates; + + // Generator of ids, globally unique and constant between re-runs. + IdGenerator & m_idGenerator; + // Color name picker of the nearest color for route RBG from our constant list of transfer colors. + ColorPicker & m_colorPicker; + + // GTFS id -> entity hash mapping. Maps GTFS id string (unique only for current feed) to the + // globally unique hash. + std::vector m_gtfsIdToHash; + + // Unique hash characterizing each GTFS feed. + std::string m_gtfsHash; + + // Unique hashes of all agencies handled by WorldFeed. + static std::unordered_set m_agencyHashes; + // Count of corrupted stops sequences which could not be projected to the shape polyline. + static size_t m_badStopSeqCount; + // Agencies which are already handled by WorldFeed and should be copied to the resulting jsons. + std::unordered_set m_agencySkipList; + + // If the feed explicitly specifies its language, we use its value. Otherwise set to default. + std::string m_feedLanguage; +}; +} // namespace transit diff --git a/transit/world_feed/world_feed_tests/CMakeLists.txt b/transit/world_feed/world_feed_tests/CMakeLists.txt new file mode 100644 index 00000000000..9b73eb7ad4b --- /dev/null +++ b/transit/world_feed/world_feed_tests/CMakeLists.txt @@ -0,0 +1,26 @@ +project(world_feed_tests) + +set( + SRC + world_feed_tests.cpp +) + +omim_add_test(${PROJECT_NAME} ${SRC}) + +omim_link_libraries( + ${PROJECT_NAME} + indexer + transit + platform + coding + geometry + base + jansson + stats_client + oauthcpp + opening_hours + world_feed + ${LIBZ} +) + +link_qt5_core(${PROJECT_NAME}) diff --git a/transit/world_feed/world_feed_tests/world_feed_tests.cpp b/transit/world_feed/world_feed_tests/world_feed_tests.cpp new file mode 100644 index 00000000000..6b3b70d75fa --- /dev/null +++ b/transit/world_feed/world_feed_tests/world_feed_tests.cpp @@ -0,0 +1,312 @@ +#include "testing/testing.hpp" + +#include "transit/world_feed/date_time_helpers.hpp" +#include "transit/world_feed/feed_helpers.hpp" + +#include "base/assert.hpp" + +#include +#include +#include +#include + +#include "3party/just_gtfs/just_gtfs.h" +#include "3party/opening_hours/opening_hours.hpp" + +using namespace transit; + +namespace +{ +std::vector GetCalendarAvailability(std::vector const & data) +{ + CHECK_EQUAL(data.size(), 7, ()); + std::vector res; + + for (auto val : data) + { + if (val == 0) + res.push_back(gtfs::CalendarAvailability::NotAvailable); + else + res.push_back(gtfs::CalendarAvailability::Available); + } + + return res; +} + +gtfs::StopTimes GetFakeStopTimes(std::vector const & transitIds) +{ + auto ids = transitIds; + std::sort(ids.begin(), ids.end()); + gtfs::StopTimes res; + for (size_t i = 0; i < ids.size(); ++i) + { + gtfs::StopTime st; + st.trip_id = ids[i]; + st.stop_sequence = i; + res.emplace_back(st); + } + return res; +} + +void TestInterval(WeekdaysInterval const & interval, size_t start, size_t end, + osmoh::RuleSequence::Modifier status) +{ + TEST_EQUAL(interval.m_start, start, ()); + TEST_EQUAL(interval.m_end, end, ()); + TEST_EQUAL(interval.m_status, status, ()); +} + +void TestExceptionIntervals(gtfs::CalendarDates const & dates, size_t intervalsCount, + std::string const & resOpeningHoursStr) +{ + osmoh::TRuleSequences rules; + GetServiceDaysExceptionsOsmoh(dates, rules); + // TEST_EQUAL(rules.size(), intervalsCount, ()); + auto const openingHours = ToString(osmoh::OpeningHours(rules)); + TEST_EQUAL(openingHours, resOpeningHoursStr, ()); +} + +void TestPlanFact(size_t planIndex, bool planInsert, std::pair const & factRes) +{ + TEST_EQUAL(factRes.first, planIndex, ()); + TEST_EQUAL(factRes.second, planInsert, ()); +} + +UNIT_TEST(Transit_GTFS_OpenCloseInterval1) +{ + auto const & intervals = GetOpenCloseIntervals(GetCalendarAvailability({1, 1, 1, 1, 1, 0, 0})); + TEST_EQUAL(intervals.size(), 2, ()); + + TestInterval(intervals[0], 0, 4, osmoh::RuleSequence::Modifier::DefaultOpen); + TestInterval(intervals[1], 5, 6, osmoh::RuleSequence::Modifier::Closed); +} + +UNIT_TEST(Transit_GTFS_OpenCloseInterval2) +{ + auto const & intervals = GetOpenCloseIntervals(GetCalendarAvailability({0, 0, 0, 0, 0, 1, 0})); + TEST_EQUAL(intervals.size(), 3, ()); + + TestInterval(intervals[0], 0, 4, osmoh::RuleSequence::Modifier::Closed); + TestInterval(intervals[1], 5, 5, osmoh::RuleSequence::Modifier::DefaultOpen); + TestInterval(intervals[2], 6, 6, osmoh::RuleSequence::Modifier::Closed); +} + +UNIT_TEST(Transit_GTFS_OpenCloseInterval3) +{ + auto const & intervals = GetOpenCloseIntervals(GetCalendarAvailability({0, 0, 0, 0, 0, 0, 1})); + TEST_EQUAL(intervals.size(), 2, ()); + + TestInterval(intervals[0], 0, 5, osmoh::RuleSequence::Modifier::Closed); + TestInterval(intervals[1], 6, 6, osmoh::RuleSequence::Modifier::DefaultOpen); +} + +UNIT_TEST(Transit_GTFS_GetTimeOsmoh) +{ + size_t const hours = 21; + size_t const minutes = 5; + size_t const seconds = 30; + gtfs::Time const timeGtfs(hours, minutes, seconds); + + auto const timeOsmoh = GetTimeOsmoh(timeGtfs); + TEST_EQUAL(timeOsmoh.GetMinutesCount(), minutes, ()); + TEST_EQUAL(timeOsmoh.GetHoursCount(), hours, ()); +} + +UNIT_TEST(Transit_GTFS_ServiceDaysExceptions1) +{ + gtfs::CalendarDates const exceptionDays{ + {"serviceId1", gtfs::Date(2015, 01, 30), gtfs::CalendarDateException::Removed}, + {"serviceId1", gtfs::Date(2015, 01, 31), gtfs::CalendarDateException::Removed}, + {"serviceId1", gtfs::Date(2015, 02, 01), gtfs::CalendarDateException::Removed}, + {"serviceId1", gtfs::Date(2015, 04, 03), gtfs::CalendarDateException::Added}}; + TestExceptionIntervals( + exceptionDays, 2 /* intervalsCount */, + "2015 Apr 03-2015 Apr 03; 2015 Jan 30-2015 Feb 01 closed" /* resOpeningHoursStr */); +} + +UNIT_TEST(Transit_GTFS_ServiceDaysExceptions2) +{ + gtfs::CalendarDates const exceptionDays{ + {"serviceId2", gtfs::Date(1999, 11, 14), gtfs::CalendarDateException::Removed}}; + TestExceptionIntervals(exceptionDays, 1 /* intervalsCount */, + "1999 Nov 14-1999 Nov 14 closed" /* resOpeningHoursStr */); +} + +UNIT_TEST(Transit_GTFS_ServiceDaysExceptions3) +{ + gtfs::CalendarDates const exceptionDays{ + {"serviceId2", gtfs::Date(2005, 8, 01), gtfs::CalendarDateException::Added}, + {"serviceId2", gtfs::Date(2005, 8, 12), gtfs::CalendarDateException::Added}, + {"serviceId2", gtfs::Date(2005, 10, 11), gtfs::CalendarDateException::Removed}, + {"serviceId2", gtfs::Date(2005, 10, 12), gtfs::CalendarDateException::Removed}, + {"serviceId2", gtfs::Date(2005, 10, 13), gtfs::CalendarDateException::Added}, + {"serviceId2", gtfs::Date(1999, 10, 14), gtfs::CalendarDateException::Removed}}; + TestExceptionIntervals( + exceptionDays, 2 /* intervalsCount */, + "2005 Aug 01-2005 Aug 01, 2005 Aug 12-2005 Aug 12, 2005 Oct 13-2005 Oct 13; 2005 Oct 11-2005 " + "Oct 12, 1999 Oct 14-1999 Oct 14 closed" /* resOpeningHoursStr */); +} + +UNIT_TEST(Transit_GTFS_FindStopTimesByTransitId) +{ + auto const allStopTimes = GetFakeStopTimes({"4", "5", "6", "2", "10", "2", "2", "6"}); + auto const stopTimes1 = GetStopTimesForTrip(allStopTimes, "2"); + TEST_EQUAL(stopTimes1.size(), 3, ()); + + auto const stopTimes10 = GetStopTimesForTrip(allStopTimes, "10"); + TEST_EQUAL(stopTimes10.size(), 1, ()); + + auto const stopTimes6 = GetStopTimesForTrip(allStopTimes, "6"); + TEST_EQUAL(stopTimes6.size(), 2, ()); + + auto const stopTimesNonExistent1 = GetStopTimesForTrip(allStopTimes, "11"); + TEST(stopTimesNonExistent1.empty(), ()); + + auto const stopTimesNonExistent2 = GetStopTimesForTrip(allStopTimes, "1"); + TEST(stopTimesNonExistent1.empty(), ()); +} + +UNIT_TEST(Transit_GTFS_FindStopTimesByTransitId2) +{ + auto const allStopTimes = GetFakeStopTimes({"28", "28", "28", "28"}); + auto const stopTimes = GetStopTimesForTrip(allStopTimes, "28"); + TEST_EQUAL(stopTimes.size(), 4, ()); + + auto const stopTimesNonExistent = GetStopTimesForTrip(allStopTimes, "3"); + TEST(stopTimesNonExistent.empty(), ()); +} + +// Stops are marked as *, points on polyline as +. Points have indexes, stops have letters. +// +// *A +// +// +----+---------------+----------------------+ +// 0 1 2 3 +// +// *B *C +// +UNIT_TEST(Transit_GTFS_ProjectStopToLine_Simple) +{ + double const y = 0.0002; + std::vector shape{{0.001, y}, {0.0015, y}, {0.004, y}, {0.005, y}}; + + m2::PointD const point_A{0.0012, 0.0003}; + m2::PointD const point_B{0.00499, 0.0001}; + m2::PointD const point_C{0.005, 0.0001}; + + // Test that point_A is projected between two existing polyline points and the new point is + // added in the place of its projection. + TestPlanFact(1 /* planIndex */, true /* planInsert */, + PrepareNearestPointOnTrack(point_A, 0 /* startIndex */, shape)); + + TEST_EQUAL(shape.size(), 5, ()); + TEST_EQUAL(shape[1 /* expectedIndex */], m2::PointD(point_A.x, y), ()); + + // Test that repeated point_A projection to the polyline doesn't lead to the second insertion. + // Expected point projection index is the same. + // But this projection is not inserted (it is already present). + TestPlanFact(1 /* planIndex */, false /* planInsert */, + PrepareNearestPointOnTrack(point_A, 0 /* startIndex */, shape)); + // So the shape size remains the same. + TEST_EQUAL(shape.size(), 5, ()); + + // Test that point_B insertion leads to addition of the new projection to the shape. + TestPlanFact(4, true, PrepareNearestPointOnTrack(point_B, 1 /* startIndex */, shape)); + + // Test that point_C insertion does not lead to the addition of the new projection. + TestPlanFact(5, false, PrepareNearestPointOnTrack(point_C, 4 /* startIndex */, shape)); +} + +// Stop is on approximately the same distance from the segment (0, 1) and segment (1, 2). +// Its projection index and projection coordinate depend on the |startIndex| parameter. +// +// 1 +----------+ 2 +// | +// | *A +// | +// 0 + +// +UNIT_TEST(Transit_GTFS_ProjectStopToLine_DifferentStartIndexes) +{ + std::vector const referenceShape{{0.001, 0.001}, {0.001, 0.002}, {0.003, 0.002}}; + m2::PointD const point_A{0.0015, 0.0015}; + + // Test for |startIndex| = 0. + { + auto shape = referenceShape; + TestPlanFact(1, true, PrepareNearestPointOnTrack(point_A, 0 /* startIndex */, shape)); + TEST_EQUAL(shape.size(), 4, ()); + TEST_EQUAL(shape[1 /* expectedIndex */], m2::PointD(0.001, point_A.y), ()); + } + + // Test for |startIndex| = 1. + { + auto shape = referenceShape; + TestPlanFact(2, true, PrepareNearestPointOnTrack(point_A, 1 /* startIndex */, shape)); + TEST_EQUAL(shape.size(), 4, ()); + TEST_EQUAL(shape[2 /* expectedIndex */], m2::PointD(point_A.x, 0.002), ()); + } +} + +// Real-life example of stop being closer to the other side of the route (4, 5) then to its real +// destination (0, 1). +// We handle this type of situations by using constant max distance of departing from this stop +// on the polyline in |PrepareNearestPointOnTrack()|. +// +// 5 4 +// +--------------------------------+---------------------------------------+ 3 +// | +// /+-------------------------------------------------+ 2 +// *A / 1 +// / +// + 0 +// +UNIT_TEST(Transit_GTFS_ProjectStopToLine_MaxDistance) +{ + std::vector shape{{0.002, 0.001}, {0.003, 0.003}, {0.010, 0.003}, + {0.010, 0.0031}, {0.005, 0.0031}, {0.001, 0.0031}}; + m2::PointD const point_A{0.0028, 0.0029}; + TestPlanFact(1, true, PrepareNearestPointOnTrack(point_A, 0 /* startIndex */, shape)); +} + +// Complex shape with multiple points on it and multiple stops for projection. +// +// +-----+ +// C* / \ +// /+\ / \ *D +// + / \*/ \ +// / + +// / | *E +// + +-----+ +// | | +// | | +// +---+\ +-----+ +// \ | +// B* + | +// A* \ +---------+ +// + | +// | + +// + *F +// +UNIT_TEST(Transit_GTFS_ProjectStopToLine_NearCircle) +{ + std::vector shape{ + {0.003, 0.001}, {0.003, 0.0015}, {0.0025, 0.002}, {0.002, 0.0025}, {0.001, 0.0025}, + {0.001, 0.0035}, {0.0015, 0.0045}, {0.0025, 0.005}, {0.0035, 0.0045}, {0.004, 0.0055}, + {0.0055, 0.0055}, {0.0065, 0.0045}, {0.0065, 0.0035}, {0.0075, 0.0035}, {0.0075, 0.0025}, + {0.0065, 0.0025}, {0.0065, 0.0015}, {0.004, 0.0015}, {0.004, 0.001}}; + + m2::PointD const point_A{0.0024, 0.0018}; + m2::PointD const point_B{0.002499, 0.00199}; + m2::PointD const point_C{0.0036, 0.0049}; + m2::PointD const point_D{0.0063, 0.005}; + m2::PointD const point_E{0.008, 0.004}; + m2::PointD const point_F{0.0047, 0.0005}; + TestPlanFact(2, true, PrepareNearestPointOnTrack(point_A, 0 /* startIndex */, shape)); + TestPlanFact(3, false, PrepareNearestPointOnTrack(point_B, 2 /* startIndex */, shape)); + TestPlanFact(10, true, PrepareNearestPointOnTrack(point_C, 3 /* startIndex */, shape)); + TestPlanFact(12, false, PrepareNearestPointOnTrack(point_D, 10 /* startIndex */, shape)); + TestPlanFact(14, true, PrepareNearestPointOnTrack(point_E, 12 /* startIndex */, shape)); + TestPlanFact(20, true, PrepareNearestPointOnTrack(point_F, 14 /* startIndex */, shape)); +} +} // namespace