Incorrect filtering results when `collect_statistics = true` and column statistics are partially disabled

[xhwhis]

Describe the bug

When datafusion.execution.collect_statistics = true is enabled, and Parquet statistics are only partially collected (i.e., some columns have statistics disabled), filters using IS NULL or IS NOT NULL on the non-statistics columns produce incorrect results.
In this case, queries that should return rows instead return no results, even though the data clearly exists in the Parquet file.

To Reproduce

use DataFusion CLI v50.2.0

> set datafusion.execution.collect_statistics = true;
0 row(s) fetched.
Elapsed 0.001 seconds.

> COPY (
  SELECT
    column1 as id,
    column2 as value,
  FROM (VALUES (1, 10), (2, 20), (3, NULL))
)
TO 'test/data.parquet'
OPTIONS (
  'statistics_enabled::value' 'none',
);
+-------+
| count |
+-------+
| 3     |
+-------+
1 row(s) fetched.
Elapsed 0.008 seconds.

> CREATE EXTERNAL TABLE t (
  id INT,
  value INT,
)
STORED AS PARQUET
LOCATION 'test';
0 row(s) fetched.
Elapsed 0.001 seconds.

> select * from t where value is null;
+----+-------+
| id | value |
+----+-------+
+----+-------+
0 row(s) fetched.
Elapsed 0.007 seconds.

> select * from 'test/data.parquet' where value is not null;
+----+-------+
| id | value |
+----+-------+
+----+-------+
0 row(s) fetched.
Elapsed 0.002 seconds.

> select * from 'test/data.parquet';
+----+-------+
| id | value |
+----+-------+
| 1  | 10    |
| 2  | 20    |
| 3  | NULL  |
+----+-------+
3 row(s) fetched.
Elapsed 0.002 seconds.

> select * from t;
+----+-------+
| id | value |
+----+-------+
| 1  | 10    |
| 2  | 20    |
| 3  | NULL  |
+----+-------+
3 row(s) fetched.
Elapsed 0.002 seconds.

select path_in_schema,stats_min,stats_max,stats_null_count,stats_min_value,stats_max_value from parquet_metadata('test/data.parquet');
+----------------+-----------+-----------+------------------+-----------------+-----------------+
| path_in_schema | stats_min | stats_max | stats_null_count | stats_min_value | stats_max_value |
+----------------+-----------+-----------+------------------+-----------------+-----------------+
| "id"           | 1         | 3         | 0                | 1               | 3               |
| "value"        | NULL      | NULL      | NULL             | NULL            | NULL            |
+----------------+-----------+-----------+------------------+-----------------+-----------------+
2 row(s) fetched.
Elapsed 0.002 seconds.

Expected behavior

Filtering with IS NULL / IS NOT NULL should still correctly evaluate against the file data even if statistics for a column are missing.

Additional context

No response

[xhwhis]

The issue likely comes from

datafusion/datafusion/datasource-parquet/src/metadata.rs

Lines 247 to 322 in 408e1e4

	let mut has_statistics = false;
	let mut num_rows = 0_usize;
	let mut total_byte_size = 0_usize;
	for row_group_meta in row_groups_metadata {
	num_rows += row_group_meta.num_rows() as usize;
	total_byte_size += row_group_meta.total_byte_size() as usize;
	if !has_statistics {
	has_statistics = row_group_meta
	.columns()
	.iter()
	.any(|column| column.statistics().is_some());
	}
	}
	statistics.num_rows = Precision::Exact(num_rows);
	statistics.total_byte_size = Precision::Exact(total_byte_size);
	let file_metadata = metadata.file_metadata();
	let mut file_schema = parquet_to_arrow_schema(
	file_metadata.schema_descr(),
	file_metadata.key_value_metadata(),
	)?;
	if let Some(merged) = apply_file_schema_type_coercions(table_schema, &file_schema)
	{
	file_schema = merged;
	}
	statistics.column_statistics = if has_statistics {
	let (mut max_accs, mut min_accs) = create_max_min_accs(table_schema);
	let mut null_counts_array =
	vec![Precision::Exact(0); table_schema.fields().len()];
	let mut is_max_value_exact = vec![Some(true); table_schema.fields().len()];
	let mut is_min_value_exact = vec![Some(true); table_schema.fields().len()];
	table_schema
	.fields()
	.iter()
	.enumerate()
	.for_each(|(idx, field)| {
	match StatisticsConverter::try_new(
	field.name(),
	&file_schema,
	file_metadata.schema_descr(),
	) {
	Ok(stats_converter) => {
	let mut accumulators = StatisticsAccumulators {
	min_accs: &mut min_accs,
	max_accs: &mut max_accs,
	null_counts_array: &mut null_counts_array,
	is_min_value_exact: &mut is_min_value_exact,
	is_max_value_exact: &mut is_max_value_exact,
	};
	summarize_min_max_null_counts(
	&mut accumulators,
	idx,
	num_rows,
	&stats_converter,
	row_groups_metadata,
	)
	.ok();
	}
	Err(e) => {
	debug!("Failed to create statistics converter: {e}");
	null_counts_array[idx] = Precision::Exact(num_rows);
	}
	}
	});
	get_col_stats(
	table_schema,
	null_counts_array,
	&mut max_accs,
	&mut min_accs,
	&mut is_max_value_exact,
	&mut is_min_value_exact,
	)

When one column has statistics, the code processes all columns’ statistics. For columns with statistics disabled, their stats are recorded as NULL instead of being marked as Absent, which causes incorrect predicate pruning on IS NULL / IS NOT NULL filters.