Merge pull request #925 from hove-io/feature/admin_regions

[feature] Add administrative regions

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 authors = ["Hove <core@hove.com>", "Guillaume Pinot <texitoi@texitoi.eu>"]
 name = "transit_model"
-version = "0.57.2"
+version = "0.58.0"
 license = "AGPL-3.0-only"
 description = "Transit data management"
 repository = "https://github.com/hove-io/transit_model"

src/gtfs/read.rs

@@ -119,6 +119,7 @@ impl TryFrom<Stop> for objects::StopArea {
             geometry_id: None,
             level_id: stop.level_id,
             equipment_id: None,
+            address_id: None,
         };
         Ok(stop_area)
     }
@@ -214,6 +215,7 @@ impl TryFrom<Stop> for objects::StopLocation {
             equipment_id: None,
             level_id: stop.level_id,
             stop_type: stop.location_type.into(),
+            address_id: None,
         };
         Ok(stop_location)
     }

src/gtfs/write.rs

@@ -761,6 +761,7 @@ mod tests {
             geometry_id: None,
             equipment_id: Some("1".to_string()),
             level_id: None,
+            address_id: None,
         };
 
         let expected = Stop {
@@ -1020,6 +1021,7 @@ mod tests {
             geometry_id: None,
             level_id: Some("level0".to_string()),
             equipment_id: None,
+            address_id: None,
         });
         let mut sp_codes: BTreeSet<(String, String)> = BTreeSet::new();
         sp_codes.insert(("sp name 1".to_string(), "sp_code_1".to_string()));

src/lib.rs

@@ -81,7 +81,7 @@ pub mod vptranslator;
 pub(crate) const STOP_TIMES_INIT_CAPACITY: usize = 50;
 
 /// Current version of the NTFS format
-pub const NTFS_VERSION: &str = "0.13.0";
+pub const NTFS_VERSION: &str = "0.14.0";
 
 /// The max distance in meters to compute the transfer
 pub const TRANSFER_MAX_DISTANCE: &str = "300";

src/model.rs

@@ -116,6 +116,7 @@ pub struct Collections {
     pub grid_periods: Collection<GridPeriod>,
     pub grid_rel_calendar_line: Collection<GridRelCalendarLine>,
     pub addresses: CollectionWithId<Address>,
+    pub administrative_regions: CollectionWithId<AdministrativeRegion>,
     pub occupancies: Collection<Occupancy>,
 }
 
@@ -309,6 +310,9 @@ impl Collections {
                     equipments_used.insert(equipment_id.clone());
                 }
                 comments_used.extend(&mut sl.comment_links.iter().map(|cl| cl.to_string()));
+                if let Some(address_id) = &sl.address_id {
+                    addresses_used.insert(address_id.clone());
+                }
                 true
             })
             .collect::<Vec<_>>();
@@ -418,6 +422,9 @@ impl Collections {
                         equipments_used.insert(equipment_id.clone());
                     }
                     comments_used.extend(&mut sa.comment_links.iter().map(|cl| cl.to_string()));
+                    if let Some(address_id) = &sa.address_id {
+                        addresses_used.insert(address_id.clone());
+                    }
                     true
                 } else {
                     log_object_removed("Stop Area", &sa.id);
@@ -556,6 +563,26 @@ impl Collections {
         self.calendars.retain(|c| calendars_used.contains(&c.id));
         self.addresses
             .retain(|address| addresses_used.contains(&address.id));
+
+        let admin_regions_used: HashSet<&str> =
+            self.addresses
+                .values()
+                .fold(HashSet::new(), |mut acc, address| {
+                    if let Some(admin_level_8_id) = address.admin_level_8_id.as_ref() {
+                        acc.insert(admin_level_8_id);
+                    }
+                    if let Some(admin_level_9_id) = address.admin_level_9_id.as_ref() {
+                        acc.insert(admin_level_9_id);
+                    }
+                    if let Some(admin_level_10_id) = address.admin_level_10_id.as_ref() {
+                        acc.insert(admin_level_10_id);
+                    }
+
+                    acc
+                });
+
+        self.administrative_regions
+            .retain(|admin| admin_regions_used.contains(admin.id.as_str()));
         self.admin_stations
             .retain(|admin_station| stop_area_ids_used.contains(&admin_station.stop_id));
         self.occupancies.retain(|occupancy| {

src/ntfs/mod.rs

@@ -317,6 +317,10 @@ where
         grid_periods: make_opt_collection(file_handler, "grid_periods.txt")?,
         grid_rel_calendar_line: make_opt_collection(file_handler, "grid_rel_calendar_line.txt")?,
         addresses: make_opt_collection_with_id(file_handler, "addresses.txt")?,
+        administrative_regions: make_opt_collection_with_id(
+            file_handler,
+            "administrative_regions.txt",
+        )?,
         ..Default::default()
     };
     manage_calendars(file_handler, &mut collections)?;
@@ -407,6 +411,11 @@ pub fn write<P: AsRef<path::Path>>(
     write_collection_with_id(path, "pathways.txt", &model.pathways)?;
     write_collection_with_id(path, "levels.txt", &model.levels)?;
     write_collection_with_id(path, "addresses.txt", &model.addresses)?;
+    write_collection_with_id(
+        path,
+        "administrative_regions.txt",
+        &model.administrative_regions,
+    )?;
     write_collection(path, "occupancies.txt", &model.occupancies)?;
 
     Ok(())
@@ -515,7 +524,7 @@ mod tests {
                     ("feed_end_date".to_string(), "20180131".to_string()),
                     ("feed_publisher_name".to_string(), "Nicaragua".to_string()),
                     ("feed_start_date".to_string(), "20180130".to_string()),
-                    ("ntfs_version".to_string(), "0.13.0".to_string()),
+                    ("ntfs_version".to_string(), "0.14.0".to_string()),
                     ("tartare_platform".to_string(), "dev".to_string()),
                 ],
                 collections
@@ -1012,6 +1021,7 @@ mod tests {
                 geometry_id: None,
                 equipment_id: None,
                 level_id: None,
+                address_id: None,
             },
             StopArea {
                 id: "sa_1".to_string(),
@@ -1028,6 +1038,7 @@ mod tests {
                 geometry_id: Some("geometry_3".to_string()),
                 equipment_id: Some("equipment_1".to_string()),
                 level_id: Some("level2".to_string()),
+                address_id: None,
             },
         ])
         .unwrap();
@@ -1117,6 +1128,7 @@ mod tests {
             geometry_id: None,
             equipment_id: None,
             level_id: Some("level1".to_string()),
+            address_id: None,
         });
 
         let stop_locations: CollectionWithId<StopLocation> = CollectionWithId::default();

src/ntfs/read.rs

@@ -54,6 +54,7 @@ impl TryFrom<Stop> for StopArea {
             geometry_id: stop.geometry_id,
             equipment_id: stop.equipment_id,
             level_id: stop.level_id,
+            address_id: stop.address_id,
         };
         Ok(stop_area)
     }
@@ -148,6 +149,7 @@ impl TryFrom<Stop> for StopLocation {
             equipment_id: stop.equipment_id,
             stop_type: stop.location_type.clone().into(),
             level_id: stop.level_id,
+            address_id: stop.address_id,
         };
         Ok(stop_location)
     }

src/ntfs/write.rs

@@ -543,7 +543,7 @@ pub fn write_stops(
                 geometry_id: sl.geometry_id.clone(),
                 level_id: sl.level_id.clone(),
                 platform_code: None,
-                address_id: None,
+                address_id: sl.address_id.clone(),
             })?;
         }
         Ok(())
@@ -595,7 +595,7 @@ pub fn write_stops(
             geometry_id: sa.geometry_id.clone(),
             level_id: sa.level_id.clone(),
             platform_code: None,
-            address_id: None,
+            address_id: sa.address_id.clone(),
         })
         .with_context(|| format!("Error reading {:?}", path))?;
     }

src/objects.rs

@@ -1070,6 +1070,7 @@ pub struct StopArea {
     pub geometry_id: Option<String>,
     pub equipment_id: Option<String>,
     pub level_id: Option<String>,
+    pub address_id: Option<String>,
 }
 impl_id!(StopArea);
 
@@ -1087,6 +1088,7 @@ impl From<StopPoint> for StopArea {
             geometry_id: None,
             equipment_id: None,
             level_id: None,
+            address_id: None,
         }
     }
 }
@@ -1208,6 +1210,7 @@ pub struct StopLocation {
     pub level_id: Option<String>,
     #[serde(skip)]
     pub stop_type: StopType,
+    pub address_id: Option<String>,
 }
 impl_id!(StopLocation);
 impl_comment_links!(StopLocation);
@@ -1899,10 +1902,35 @@ pub struct Address {
     pub id: String,
     pub street_name: String,
     pub house_number: Option<String>,
+    pub admin_level_8_id: Option<String>,
+    pub admin_level_9_id: Option<String>,
+    pub admin_level_10_id: Option<String>,
 }
 
 impl_id!(Address);
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct AdministrativeRegion {
+    #[serde(rename = "admin_id")]
+    pub id: String,
+    #[serde(rename = "admin_insee")]
+    pub insee: Option<String>,
+    #[serde(rename = "admin_level")]
+    pub level: Option<u32>,
+    #[serde(rename = "admin_name")]
+    pub name: Option<String>,
+    #[serde(rename = "admin_label")]
+    pub label: Option<String>,
+    #[serde(rename = "admin_zip_codes")]
+    pub zip_codes: Option<String>,
+    #[serde(rename = "admin_lon")]
+    pub lon: Option<f64>,
+    #[serde(rename = "admin_lat")]
+    pub lat: Option<f64>,
+}
+
+impl_id!(AdministrativeRegion);
+
 impl AddPrefix for Address {
     fn prefix(&mut self, prefix_conf: &PrefixConfiguration) {
         self.id = prefix_conf.referential_prefix(self.id.as_str());

tests/fixtures/gtfs2ntfs/full_output/feed_infos.txt

@@ -7,6 +7,6 @@ feed_license,DefaultDatasourceLicense
 feed_license_url,http://www.default-datasource-website.com
 feed_publisher_name,DefaultContributorName
 feed_start_date,20180101
-ntfs_version,0.13.0
+ntfs_version,0.14.0
 tartare_contributor_id,DefaultContributorId
 tartare_platform,dev

tests/fixtures/gtfs2ntfs/minimal/output/feed_infos.txt

@@ -4,4 +4,4 @@ feed_creation_time,17:19:00
 feed_creation_datetime,2019-04-03T17:19:00+00:00
 feed_end_date,20180106
 feed_start_date,20180101
-ntfs_version,0.13.0
+ntfs_version,0.14.0

tests/fixtures/gtfs2ntfs/routes_comments/output/feed_infos.txt

@@ -4,4 +4,4 @@ feed_creation_time,17:19:00
 feed_creation_datetime,2019-04-03T17:19:00+00:00
 feed_end_date,20180106
 feed_start_date,20180101
-ntfs_version,0.13.0
+ntfs_version,0.14.0

tests/fixtures/gtfs2ntfs/routes_comments/output_as_lines/feed_infos.txt

@@ -4,4 +4,4 @@ feed_creation_time,17:19:00
 feed_creation_datetime,2019-04-03T17:19:00+00:00
 feed_end_date,20180106
 feed_start_date,20180101
-ntfs_version,0.13.0
+ntfs_version,0.14.0

tests/fixtures/minimal_ntfs/addresses.txt

@@ -1,5 +1,5 @@
-address_id,street_name,house_number
-1,rue de Bercy,Face au 9
-2,nation,
-3,boulevard Montparnasse,23
-4,must be sanitized,
+address_id,street_name,house_number,admin_level_8_id,admin_level_9_id,admin_level_10_id
+1,rue de Bercy,Face au 9,admin:fr:75056,admin:osm:relation:9525,admin:osm:relation:2171718
+2,nation,,,,
+3,boulevard Montparnasse,23,,,,
+4,must be sanitized,,,,

tests/fixtures/minimal_ntfs/administrative_regions.txt

@@ -0,0 +1,5 @@
+admin_id,admin_insee,admin_level,admin_name,admin_label,admin_zip_codes,admin_lon,admin_lat
+admin:fr:75056,75056,10,Paris,Paris (75000-75116),75000;75001;75002;75003;75004;75005;75006;75007;75008;75009;75010;75011;75012;75013;75014;75015;75016;75017;75018;75019;75020;75116,2.3483915,48.8534951
+admin:osm:relation:9525,75112,9,Paris 12e Arrondissement,Paris 12e Arrondissement (75012),75012,2.3957517,48.8396154
+admin:osm:relation:2171718,,10,Quartier de Bercy,Quartier de Bercy (75012),75012,2.386208,48.835201
+sanitize_me,,10,sanitize_me,sanitize_me,75012,2.386208,48.835201

tests/fixtures/ntfs2ntfs/stops/addresses.txt

@@ -1,4 +1,4 @@
-address_id,street_name,house_number
-1,rue de Bercy,Face au 9
-2,nation,
-3,boulevard Montparnasse,23
+address_id,street_name,house_number,admin_level_8_id,admin_level_9_id,admin_level_10_id
+1,rue de Bercy,Face au 9,admin:fr:75056,admin:osm:relation:9525,admin:osm:relation:2171718
+2,nation,,,,
+3,boulevard Montparnasse,23,,,

tests/fixtures/ntfs2ntfs/stops/administrative_regions.txt

@@ -0,0 +1,4 @@
+admin_id,admin_insee,admin_level,admin_name,admin_label,admin_zip_codes,admin_lon,admin_lat
+admin:fr:75056,75056,10,Paris,Paris (75000-75116),75000;75001;75002;75003;75004;75005;75006;75007;75008;75009;75010;75011;75012;75013;75014;75015;75016;75017;75018;75019;75020;75116,2.3483915,48.8534951
+admin:osm:relation:9525,75112,9,Paris 12e Arrondissement,Paris 12e Arrondissement (75012),75012,2.3957517,48.8396154
+admin:osm:relation:2171718,,10,Quartier de Bercy,Quartier de Bercy (75012),75012,2.386208,48.835201

tests/fixtures/restrict-validity-period/output/feed_infos.txt

@@ -4,4 +4,4 @@ feed_creation_time,17:19:00
 feed_creation_datetime,2019-04-03T17:19:00+00:00
 feed_end_date,20180805
 feed_start_date,20180501
-ntfs_version,0.13.0
+ntfs_version,0.14.0

tests/read_ntfs.rs

@@ -141,7 +141,12 @@ fn ntfs_stops_output() {
         transit_model::ntfs::write(&ntm, output_dir, get_test_datetime()).unwrap();
         compare_output_dir_with_expected(
             output_dir,
-            Some(vec!["stops.txt", "stop_times.txt", "addresses.txt"]),
+            Some(vec![
+                "stops.txt",
+                "stop_times.txt",
+                "addresses.txt",
+                "administrative_regions.txt",
+            ]),
             "tests/fixtures/ntfs2ntfs/stops",
         );
     });