Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Faster Null Path Selection in ArrayData Equality #2676

Merged
merged 2 commits into from
Sep 7, 2022
Merged

Faster Null Path Selection in ArrayData Equality #2676

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
740de70
Use contains_nulls if we don't need null count
dhruv9vats Sep 7, 2022
3ab3322
Inline function call
dhruv9vats Sep 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 3 additions & 4 deletions arrow/src/array/equal/decimal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use crate::array::{data::count_nulls, ArrayData};
use crate::array::{data::contains_nulls, ArrayData};
use crate::datatypes::DataType;
use crate::util::bit_util::get_bit;

Expand All @@ -37,10 +37,9 @@ pub(super) fn decimal_equal(
let lhs_values = &lhs.buffers()[0].as_slice()[lhs.offset() * size..];
let rhs_values = &rhs.buffers()[0].as_slice()[rhs.offset() * size..];

let lhs_null_count = count_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);
let rhs_null_count = count_nulls(rhs.null_buffer(), rhs_start + rhs.offset(), len);
let has_nulls = contains_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);

if lhs_null_count == 0 && rhs_null_count == 0 {
dhruv9vats marked this conversation as resolved.
Show resolved Hide resolved
if !has_nulls {
dhruv9vats marked this conversation as resolved.
Show resolved Hide resolved
equal_len(
lhs_values,
rhs_values,
Expand Down
7 changes: 3 additions & 4 deletions arrow/src/array/equal/dictionary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use crate::array::{data::count_nulls, ArrayData};
use crate::array::{data::contains_nulls, ArrayData};
use crate::datatypes::ArrowNativeType;
use crate::util::bit_util::get_bit;

Expand All @@ -34,10 +34,9 @@ pub(super) fn dictionary_equal<T: ArrowNativeType>(
let lhs_values = &lhs.child_data()[0];
let rhs_values = &rhs.child_data()[0];

let lhs_null_count = count_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);
let rhs_null_count = count_nulls(rhs.null_buffer(), rhs_start + rhs.offset(), len);
let has_nulls = contains_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);

if lhs_null_count == 0 && rhs_null_count == 0 {
if !has_nulls {
(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;
Expand Down
7 changes: 3 additions & 4 deletions arrow/src/array/equal/fixed_binary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use crate::array::{data::count_nulls, ArrayData};
use crate::array::{data::contains_nulls, ArrayData};
use crate::datatypes::DataType;
use crate::util::bit_util::get_bit;

Expand All @@ -36,10 +36,9 @@ pub(super) fn fixed_binary_equal(
let lhs_values = &lhs.buffers()[0].as_slice()[lhs.offset() * size..];
let rhs_values = &rhs.buffers()[0].as_slice()[rhs.offset() * size..];

let lhs_null_count = count_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);
let rhs_null_count = count_nulls(rhs.null_buffer(), rhs_start + rhs.offset(), len);
let has_nulls = contains_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);

if lhs_null_count == 0 && rhs_null_count == 0 {
if !has_nulls {
equal_len(
lhs_values,
rhs_values,
Expand Down
7 changes: 3 additions & 4 deletions arrow/src/array/equal/fixed_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use crate::array::{data::count_nulls, ArrayData};
use crate::array::{data::contains_nulls, ArrayData};
use crate::datatypes::DataType;
use crate::util::bit_util::get_bit;

Expand All @@ -36,10 +36,9 @@ pub(super) fn fixed_list_equal(
let lhs_values = &lhs.child_data()[0];
let rhs_values = &rhs.child_data()[0];

let lhs_null_count = count_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);
let rhs_null_count = count_nulls(rhs.null_buffer(), rhs_start + rhs.offset(), len);
let has_nulls = contains_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);

if lhs_null_count == 0 && rhs_null_count == 0 {
if !has_nulls {
equal_range(
lhs_values,
rhs_values,
Expand Down
7 changes: 3 additions & 4 deletions arrow/src/array/equal/primitive.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

use std::mem::size_of;

use crate::array::{data::count_nulls, ArrayData};
use crate::array::{data::contains_nulls, ArrayData};
use crate::util::bit_util::get_bit;

use super::utils::equal_len;
Expand All @@ -33,10 +33,9 @@ pub(super) fn primitive_equal<T>(
let lhs_values = &lhs.buffers()[0].as_slice()[lhs.offset() * byte_width..];
let rhs_values = &rhs.buffers()[0].as_slice()[rhs.offset() * byte_width..];

let lhs_null_count = count_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);
let rhs_null_count = count_nulls(rhs.null_buffer(), rhs_start + rhs.offset(), len);
let has_nulls = contains_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);

if lhs_null_count == 0 && rhs_null_count == 0 {
if !has_nulls {
// without nulls, we just need to compare slices
equal_len(
lhs_values,
Expand Down
8 changes: 3 additions & 5 deletions arrow/src/array/equal/structure.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use crate::{array::data::count_nulls, array::ArrayData, util::bit_util::get_bit};
use crate::{array::data::contains_nulls, array::ArrayData, util::bit_util::get_bit};

use super::equal_range;

Expand Down Expand Up @@ -43,11 +43,9 @@ pub(super) fn struct_equal(
rhs_start: usize,
len: usize,
) -> bool {
// we have to recalculate null counts from the null buffers
let lhs_null_count = count_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);
let rhs_null_count = count_nulls(rhs.null_buffer(), rhs_start + rhs.offset(), len);
let has_nulls = contains_nulls(lhs.null_buffer(), lhs_start + lhs.offset(), len);

if lhs_null_count == 0 && rhs_null_count == 0 {
if !has_nulls {
equal_child_values(lhs, rhs, lhs_start, rhs_start, len)
} else {
// get a ref of the null buffer bytes, to use in testing for nullness
Expand Down