Skip to content

feat: Implement datafusion-spark sequence function #20955

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
erenavsarogullari wants to merge 6 commits into
base: main
Choose a base branch
from
Open

feat: Implement datafusion-spark sequence function #20955

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f571da9
feat: Implement datafusion-spark sequence function
erenavsarogullari Feb 22, 2026
aa579a7
Add step parameter
erenavsarogullari Mar 24, 2026
a0ed965
Apply refactoring to fix array.slt test failures
erenavsarogullari Apr 6, 2026
a4753b8
Fix CI test failures
erenavsarogullari Apr 6, 2026
f922126
Address review comment
erenavsarogullari Apr 16, 2026
df57cff
Add more test coverage
erenavsarogullari May 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions datafusion/functions-nested/src/range.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ impl Range {
}

/// Generate `generate_series()` function which includes upper bound.
fn generate_series() -> Self {
pub fn generate_series() -> Self {
Self {
signature: Self::defined_signature(),
include_upper_bound: true,
Expand Down Expand Up @@ -296,7 +296,7 @@ impl Range {
/// gen_range(3) => [0, 1, 2]
/// gen_range(1, 4) => [1, 2, 3]
/// gen_range(1, 7, 2) => [1, 3, 5]
fn gen_range_inner(&self, args: &[ArrayRef]) -> Result<ArrayRef> {
pub fn gen_range_inner(&self, args: &[ArrayRef]) -> Result<ArrayRef> {
let (start_array, stop_array, step_array) = match args {
[stop_array] => (None, as_int64_array(stop_array)?, None),
[start_array, stop_array] => (
Expand Down Expand Up @@ -353,7 +353,7 @@ impl Range {
Ok(arr)
}

fn gen_range_date(&self, args: &[ArrayRef]) -> Result<ArrayRef> {
pub fn gen_range_date(&self, args: &[ArrayRef]) -> Result<ArrayRef> {
let [start, stop, step] = take_function_args(self.name(), args)?;
let step = as_interval_mdn_array(step)?;

Expand Down Expand Up @@ -417,7 +417,7 @@ impl Range {
Ok(arr)
}

fn gen_range_timestamp(&self, args: &[ArrayRef]) -> Result<ArrayRef> {
pub fn gen_range_timestamp(&self, args: &[ArrayRef]) -> Result<ArrayRef> {
let [start, stop, step] = take_function_args(self.name(), args)?;
let step = as_interval_mdn_array(step)?;

Expand Down
8 changes: 8 additions & 0 deletions datafusion/spark/src/function/array/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

pub mod array_contains;
pub mod repeat;
pub mod sequence;
pub mod shuffle;
pub mod slice;
pub mod spark_array;
Expand All @@ -30,6 +31,7 @@ make_udf_function!(spark_array::SparkArray, array);
make_udf_function!(shuffle::SparkShuffle, shuffle);
make_udf_function!(repeat::SparkArrayRepeat, array_repeat);
make_udf_function!(slice::SparkSlice, slice);
make_udf_function!(sequence::SparkSequence, sequence);

pub mod expr_fn {
use datafusion_functions::export_functions;
Expand All @@ -55,6 +57,11 @@ pub mod expr_fn {
"Returns a slice of the array from the start index with the given length.",
array start length
));
export_functions!((
sequence,
"Returns a sequence of the array from the start index and end index.",
start stop step
));
}

pub fn functions() -> Vec<Arc<ScalarUDF>> {
Expand All @@ -63,6 +70,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
array(),
shuffle(),
array_repeat(),
sequence(),
slice(),
]
}
Loading
Loading