Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Simplified example. (#619)
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao committed Nov 20, 2021
1 parent 456accd commit 6639f89
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 120 deletions.
1 change: 1 addition & 0 deletions examples/s3/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ arrow2 = { path = "../../", default-features = false, features = ["io_parquet",
rust-s3 = { version = "0.27.0", features = ["tokio"] }
futures = "0.3"
tokio = { version = "1.0.0", features = ["macros", "rt-multi-thread"] }
range-reader = "0.1"
12 changes: 5 additions & 7 deletions examples/s3/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,9 @@ use arrow2::io::parquet::read::{
decompress, get_page_stream, page_stream_to_array, read_metadata_async,
};
use futures::{future::BoxFuture, StreamExt};
use range_reader::{RangeOutput, RangedAsyncReader};
use s3::Bucket;

mod stream;
use stream::{RangedStreamer, SeekOutput};

#[tokio::main]
async fn main() -> Result<()> {
let bucket_name = "dev-jorgecardleitao";
Expand All @@ -36,12 +34,12 @@ async fn main() -> Result<()> {
.map_err(|x| std::io::Error::new(std::io::ErrorKind::Other, x.to_string()))?;
println!("got {}/{} bytes starting at {}", data.len(), length, start);
data.truncate(length);
Ok(SeekOutput { start, data })
}) as BoxFuture<'static, std::io::Result<SeekOutput>>
Ok(RangeOutput { start, data })
}) as BoxFuture<'static, std::io::Result<RangeOutput>>
});

// at least 4kb per s3 request. Adjust as you like.
let mut reader = RangedStreamer::new(length, 4 * 1024, range_get);
// at least 4kb per s3 request. Adjust to your liking.
let mut reader = RangedAsyncReader::new(length, 4 * 1024, range_get);

let metadata = read_metadata_async(&mut reader).await?;

Expand Down
113 changes: 0 additions & 113 deletions examples/s3/src/stream.rs

This file was deleted.

0 comments on commit 6639f89

Please sign in to comment.