-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ARROW-7971: [Rust] Create rowcount utility
This utility introduces a way to count the number of rows present in one or more Parquet files. This was tested against the Parquet payloads in the `python/pyarrow/tests/data/parquet/` directory. I created this utility out of necessity, as the `parquet-tools` project introduces applications written using Java. This is a much faster alternative, and allows for multiple files to be counted at a time, rather than `parquet-tools` ability to only count one file at a time. Closes #6511 from KenSuenobu/parquet-rowcount-rust and squashes the following commits: 76ce2d2 <Ken Suenobu> Renamed variable. 794237e <Ken Suenobu> Changed code to use row_groups() from metadata as suggested by @andygrove 31ebd36 <Ken Suenobu> Removed formatted space. 819df7a <Ken Suenobu> Creation of parquet-rowcount tool to help count number of rows in a Parquet file. Authored-by: Ken Suenobu <ksuenobu@fastmail.com> Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
- Loading branch information
1 parent
4231663
commit 357eb6b
Showing
2 changed files
with
78 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
// Licensed to the Apache Software Foundation (ASF) under one | ||
// or more contributor license agreements. See the NOTICE file | ||
// distributed with this work for additional information | ||
// regarding copyright ownership. The ASF licenses this file | ||
// to you under the Apache License, Version 2.0 (the | ||
// "License"); you may not use this file except in compliance | ||
// with the License. You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, | ||
// software distributed under the License is distributed on an | ||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
// KIND, either express or implied. See the License for the | ||
// specific language governing permissions and limitations | ||
// under the License. | ||
|
||
//! Binary file to return the number of rows found from Parquet file(s). | ||
//! | ||
//! # Install | ||
//! | ||
//! `parquet-rowcount` can be installed using `cargo`: | ||
//! ``` | ||
//! cargo install parquet | ||
//! ``` | ||
//! After this `parquet-rowcount` should be globally available: | ||
//! ``` | ||
//! parquet-rowcount XYZ.parquet | ||
//! ``` | ||
//! | ||
//! The binary can also be built from the source code and run as follows: | ||
//! ``` | ||
//! cargo run --bin parquet-rowcount XYZ.parquet ABC.parquet ZXC.parquet | ||
//! ``` | ||
//! | ||
//! # Usage | ||
//! | ||
//! ``` | ||
//! parquet-rowcount <file-path> ... | ||
//! ``` | ||
//! where `file-path` is the path to a Parquet file and `...` is any additional number of | ||
//! parquet files to count the number of rows from. | ||
//! | ||
//! Note that `parquet-rowcount` reads full file schema, no projection or filtering is | ||
//! applied. | ||
|
||
extern crate parquet; | ||
|
||
use std::{env, fs::File, path::Path, process}; | ||
|
||
use parquet::file::reader::{FileReader, SerializedFileReader}; | ||
|
||
fn main() { | ||
let args: Vec<String> = env::args().collect(); | ||
if args.len() < 2 { | ||
println!("Usage: parquet-rowcount <file-path> ..."); | ||
process::exit(1); | ||
} | ||
|
||
for i in 1..args.len() { | ||
let filename = args[i].clone(); | ||
let path = Path::new(&filename); | ||
let file = File::open(&path).unwrap(); | ||
let parquet_reader = SerializedFileReader::new(file).unwrap(); | ||
let row_group_metadata = parquet_reader.metadata().row_groups(); | ||
let mut total_num_rows = 0; | ||
|
||
for group_metadata in row_group_metadata { | ||
total_num_rows += group_metadata.num_rows(); | ||
} | ||
|
||
eprintln!("File {}: rowcount={}", filename, total_num_rows); | ||
} | ||
} |