Skip to content

Commit

Permalink
simplier gtfs reading api
Browse files Browse the repository at this point in the history 
closes #739

hide the `Configuration` needed to read GTFS to have easier hello world and an api closer to the ntfs module.

Also add a `gtfs::read` method that detect if the GTFS is a zip or not (like `ntfs::read`).

now the canonimal way to read a gtfs is:

```rust
let model = transit_model::gtfs::read("path to a zip or directory")?;
```

For better control there is also:

```rust
let model = transit_model::gtfs::read_from_path("path to a directory")?;
```
```rust
let model = transit_model::gtfs::read_from_zip("path to a zip")?;
```
or even a `gtfs::from_read` method that takes a reader (cf doc).

When a configuration is need a `Reader` struct is needed
```rust
let conf = gtfs::Configuration {
            read_as_line: true,
            ..Default::default()
        };
let model = transit_model::gtfs::Reader::new(conf).from(input_dir)?;
```
  • Loading branch information
@antoine-de
antoine-de committed May 7, 2021
1 parent 6f08bfd commit a2d0bcd
Show file tree
Hide file tree
Showing 6 changed files with 123 additions and 52 deletions.
3 changes: 1 addition & 2 deletions examples/gtfs_reader.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ use serde_json::json;
use transit_model::{gtfs, Result};

fn run() -> Result<()> {
let configuration = gtfs::Configuration::default();
// read GTFS from current directory
let objects = gtfs::read_from_path(".", configuration)?;
let objects = gtfs::read(".")?;
// output internal model as JSON
let json_objs = json!(objects);
println!("{}", json_objs.to_string());
Expand Down
9 changes: 1 addition & 8 deletions gtfs2netexfr/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
// <http://www.gnu.org/licenses/>.

use chrono::{DateTime, FixedOffset};
use failure::bail;
use log::info;
use slog::{slog_o, Drain};
use slog_async::OverflowStrategy;
Expand Down Expand Up @@ -114,13 +113,7 @@ fn run(opt: Opt) -> Result<()> {
..Default::default()
};

let model = if opt.input.is_file() {
transit_model::gtfs::read_from_zip(opt.input, configuration)?
} else if opt.input.is_dir() {
transit_model::gtfs::read_from_path(opt.input, configuration)?
} else {
bail!("Invalid input data: must be an existing directory or a ZIP archive");
};
let model = transit_model::gtfs::Reader::new(configuration).from(opt.input)?;

let netex_exporter = transit_model::netex_france::Exporter::new(
&model,
Expand Down
9 changes: 1 addition & 8 deletions gtfs2ntfs/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
// <http://www.gnu.org/licenses/>.

use chrono::{DateTime, FixedOffset};
use failure::bail;
use log::info;
use slog::{slog_o, Drain};
use slog_async::OverflowStrategy;
Expand Down Expand Up @@ -135,13 +134,7 @@ fn run(opt: Opt) -> Result<()> {
read_as_line: opt.read_as_line,
};

let model = if opt.input.is_file() {
transit_model::gtfs::read_from_zip(opt.input, configuration)?
} else if opt.input.is_dir() {
transit_model::gtfs::read_from_path(opt.input, configuration)?
} else {
bail!("Invalid input data: must be an existing directory or a ZIP archive");
};
let model = transit_model::gtfs::Reader::new(configuration).from(opt.input)?;

let model = generates_transfers(
model,
Expand Down
119 changes: 100 additions & 19 deletions src/gtfs/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ use crate::{
};
use chrono_tz::Tz;
use derivative::Derivative;
use failure::ResultExt;
use log::info;
use serde::{Deserialize, Serialize};
use std::{collections::BTreeMap, fmt, path::Path};
Expand Down Expand Up @@ -273,7 +274,7 @@ pub struct Configuration {
pub read_as_line: bool,
}

fn read<H>(file_handler: &mut H, configuration: Configuration) -> Result<Model>
fn read_file_handler<H>(file_handler: &mut H, configuration: Configuration) -> Result<Model>
where
for<'a> &'a mut H: read_utils::FileHandler,
{
Expand Down Expand Up @@ -334,19 +335,14 @@ where
/// files in the `path` directory.
///
/// The `Configuration` is used to control various parameters during the import.
pub fn read_from_path<P: AsRef<Path>>(p: P, configuration: Configuration) -> Result<Model> {
let mut file_handle = read_utils::PathFileHandler::new(p.as_ref().to_path_buf());
read(&mut file_handle, configuration)
pub fn read_from_path<P: AsRef<Path>>(p: P) -> Result<Model> {
Reader::default().from_path(p)
}

/// Imports a `Model` from a zip file containing the
/// [GTFS](https://gtfs.org/reference/static).
///
/// The `Configuration` is used to control various parameters during the import.
pub fn read_from_zip<P: AsRef<Path>>(p: P, configuration: Configuration) -> Result<Model> {
let reader = std::fs::File::open(p.as_ref())?;
let mut file_handler = read_utils::ZipHandler::new(reader, p)?;
read(&mut file_handler, configuration)
pub fn read_from_zip<P: AsRef<Path>>(p: P) -> Result<Model> {
Reader::default().from_zip(p)
}

/// Imports a `Model` from an object implementing `Read` and `Seek` and containing the
Expand All @@ -355,21 +351,106 @@ pub fn read_from_zip<P: AsRef<Path>>(p: P, configuration: Configuration) -> Resu
/// This method makes it possible to read from a variety of sources like read a GTFS
/// from the network.
///
/// ```
// let url = "http://some_url/gtfs.zip";
// let resp = reqwest::blocking::get(url)?; // or async call
// let data = std::io::Cursor::new(resp.bytes()?.to_vec());
// let model = transit_model::gtfs::from_read(data, &url, configuration)?;
/// ```ignore
/// let url = "http://some_url/gtfs.zip";
/// let resp = reqwest::blocking::get(url)?; // or async call
/// let data = std::io::Cursor::new(resp.bytes()?.to_vec());
/// let model = transit_model::gtfs::from_read(data, &url)?;
/// # Ok::<(), Error>(())
/// ```
///
/// The `source_name` is needed to have nicer error messages.
/// The `Configuration` is used to control various parameters during the import.
pub fn from_read<R>(reader: R, source_name: &str, configuration: Configuration) -> Result<Model>
pub fn from_read<R>(reader: R, source_name: &str) -> Result<Model>
where
R: std::io::Seek + std::io::Read,
{
let mut file_handler = read_utils::ZipHandler::new(reader, &source_name)?;
read(&mut file_handler, configuration)
Reader::default().from_reader(reader, source_name)
}

/// Imports a `Model` from the
/// [GTFS](https://gtfs.org/reference/static).
/// files in the given directory.
/// This method will try to detect if the input is a ziped archive or not.
/// If the default file type mechanism is not enough, you can use
/// [read_from_zip] or [read_from_path].
pub fn read<P: AsRef<Path>>(p: P) -> Result<Model> {
Reader::default().from(p)
}

/// Structure to configure the GTFS reading
#[derive(Default)]
pub struct Reader {
configuration: Configuration,
}

impl Reader {
/// Build a Reader with a custom configuration
pub fn new(configuration: Configuration) -> Self {
Self { configuration }
}

/// Imports a `Model` from the
/// [GTFS](https://gtfs.org/reference/static).
/// files in the given directory.
/// This method will try to detect if the input is a ziped archive or not.
/// If the default file type mechanism is not enough, you can use
/// [Reader::from_zip] or [Reader::from_path].
pub fn from(self, path: impl AsRef<Path>) -> Result<Model> {
let p = path.as_ref();
if p.is_file() {
// if it's a file, we consider it to be a zip (and an error will be returned if it is not)
Ok(self
.from_zip(p)
.with_context(|_| format!("impossible to read ziped gtfs {:?}", p))?)
} else if p.is_dir() {
Ok(self
.from_path(p)
.with_context(|_| format!("impossible to read gtfs directory from {:?}", p))?)
} else {
Err(failure::format_err!(
"file {:?} is neither a file nor a directory, cannot read a gtfs from it",
p
))
}
}

/// Imports a `Model` from a zip file containing the
/// [GTFS](https://gtfs.org/reference/static).
pub fn from_zip(self, path: impl AsRef<Path>) -> Result<Model> {
let reader = std::fs::File::open(path.as_ref())?;
let mut file_handler = read_utils::ZipHandler::new(reader, path)?;
read_file_handler(&mut file_handler, self.configuration)
}

/// Imports a `Model` from the [GTFS](https://gtfs.org/reference/static)
/// files in the `path` directory.
pub fn from_path(self, path: impl AsRef<Path>) -> Result<Model> {
let mut file_handler = read_utils::PathFileHandler::new(path.as_ref().to_path_buf());
read_file_handler(&mut file_handler, self.configuration)
}

/// Imports a `Model` from an object implementing `Read` and `Seek` and containing the
/// [GTFS](https://gtfs.org/reference/static).
///
/// This method makes it possible to read from a variety of sources like read a GTFS
/// from the network.
///
/// ```ignore
/// let url = "http://some_url/gtfs.zip";
/// let resp = reqwest::blocking::get(url)?; // or async call
/// let data = std::io::Cursor::new(resp.bytes()?.to_vec());
/// let model = transit_model::gtfs::Reader::default().from_reader(data, &url)?;
/// # Ok::<(), Error>(())
/// ```
///
/// The `source_name` is needed to have nicer error messages.
pub fn from_reader<R>(self, reader: R, source_name: &str) -> Result<Model>
where
R: std::io::Seek + std::io::Read,
{
let mut file_handler = read_utils::ZipHandler::new(reader, source_name)?;
read_file_handler(&mut file_handler, self.configuration)
}
}

#[derive(PartialOrd, Ord, Debug, Clone, Eq, PartialEq, Hash)]
Expand Down
30 changes: 19 additions & 11 deletions tests/gtfs2ntfs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ fn test_gtfs() {
on_demand_transport_comment: None,
read_as_line: false,
};
let model = transit_model::gtfs::read_from_path(input_dir, configuration).unwrap();
let model = transit_model::gtfs::Reader::new(configuration)
.from(input_dir)
.unwrap();
transit_model::ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(&path, None, "./tests/fixtures/gtfs2ntfs/full_output");
});
Expand All @@ -49,7 +51,7 @@ fn test_gtfs() {
fn test_minimal_gtfs() {
test_in_tmp_dir(|path| {
let input_dir = "./tests/fixtures/gtfs2ntfs/minimal/input";
let model = gtfs::read_from_path(input_dir, gtfs::Configuration::default()).unwrap();
let model = transit_model::gtfs::read(input_dir).unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(&path, None, "./tests/fixtures/gtfs2ntfs/minimal/output");
});
Expand All @@ -59,7 +61,7 @@ fn test_minimal_gtfs() {
fn test_gtfs_physical_modes() {
test_in_tmp_dir(|path| {
let input_dir = "./tests/fixtures/gtfs2ntfs/physical_modes/input";
let model = gtfs::read_from_path(input_dir, gtfs::Configuration::default()).unwrap();
let model = transit_model::gtfs::read(input_dir).unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(
&path,
Expand All @@ -78,7 +80,7 @@ fn test_gtfs_physical_modes() {
fn test_gtfs_remove_vjs_with_no_traffic() {
test_in_tmp_dir(|path| {
let input_dir = "./tests/fixtures/gtfs2ntfs/no_traffic/input";
let model = gtfs::read_from_path(input_dir, gtfs::Configuration::default()).unwrap();
let model = transit_model::gtfs::read(input_dir).unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(
&path,
Expand All @@ -99,7 +101,7 @@ fn test_gtfs_remove_vjs_with_no_traffic() {
fn test_minimal_ziped_gtfs() {
test_in_tmp_dir(|path| {
let input = "./tests/fixtures/ziped_gtfs/gtfs.zip";
let model = gtfs::read_from_zip(input, gtfs::Configuration::default()).unwrap();
let model = transit_model::gtfs::read(input).unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(&path, None, "./tests/fixtures/gtfs2ntfs/minimal/output");
});
Expand All @@ -109,7 +111,7 @@ fn test_minimal_ziped_gtfs() {
fn test_minimal_ziped_sub_dir_gtfs() {
test_in_tmp_dir(|path| {
let input = "./tests/fixtures/ziped_gtfs/sub_dir_gtfs.zip";
let model = gtfs::read_from_zip(input, gtfs::Configuration::default()).unwrap();
let model = transit_model::gtfs::read(input).unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(&path, None, "./tests/fixtures/gtfs2ntfs/minimal/output");
});
Expand All @@ -119,7 +121,7 @@ fn test_minimal_ziped_sub_dir_gtfs() {
fn test_minimal_ziped_sub_dir_gtfs_with_hidden_files() {
test_in_tmp_dir(|path| {
let input = "./tests/fixtures/ziped_gtfs/sub_dir_gtfs_with_hidden_files.zip";
let model = gtfs::read_from_zip(input, gtfs::Configuration::default()).unwrap();
let model = transit_model::gtfs::read(input).unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(&path, None, "./tests/fixtures/gtfs2ntfs/minimal/output");
});
Expand All @@ -142,7 +144,9 @@ fn test_minimal_gtfs_with_odt_comment() {
),
read_as_line: false,
};
let model = gtfs::read_from_path(input_dir, configuration).unwrap();
let model = transit_model::gtfs::Reader::new(configuration)
.from(input_dir)
.unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(
&path,
Expand Down Expand Up @@ -170,7 +174,9 @@ fn test_minimal_gtfs_frequencies_with_odt_comment() {
read_as_line: false,
};

let model = gtfs::read_from_path(input_dir, configuration).unwrap();
let model = transit_model::gtfs::Reader::new(configuration)
.from(input_dir)
.unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(
&path,
Expand All @@ -184,7 +190,7 @@ fn test_minimal_gtfs_frequencies_with_odt_comment() {
fn test_minimal_gtfs_with_routes_comments() {
test_in_tmp_dir(|path| {
let input_dir = "./tests/fixtures/gtfs2ntfs/routes_comments/input";
let model = gtfs::read_from_path(input_dir, gtfs::Configuration::default()).unwrap();
let model = transit_model::gtfs::read(input_dir).unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(
&path,
Expand All @@ -202,7 +208,9 @@ fn test_minimal_gtfs_with_routes_as_lines_comments() {
read_as_line: true,
..Default::default()
};
let model = gtfs::read_from_path(input_dir, configuration).unwrap();
let model = transit_model::gtfs::Reader::new(configuration)
.from(input_dir)
.unwrap();
ntfs::write(&model, path, get_test_datetime()).unwrap();
compare_output_dir_with_expected(
&path,
Expand Down
5 changes: 1 addition & 4 deletions tests/write_netex_france.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,7 @@ fn test_write_netex_france_from_ntfs() {

#[test]
fn test_write_netex_france_from_gtfs() {
let configuration = gtfs::Configuration::default();

let model =
gtfs::read_from_path("tests/fixtures/netex_france/input_gtfs", configuration).unwrap();
let model = gtfs::read("tests/fixtures/netex_france/input_gtfs").unwrap();
test_write_netex_france(model);
}

Expand Down

0 comments on commit a2d0bcd

Please sign in to comment.