Upgrade to Datafusion 51 #1311
Upgrade to Datafusion 51 #1311
Conversation
| "List(nullable Int64)", | ||
| "List(nullable Int64)", | ||
| "List(nullable Int64)", | ||
| "List(nullable Int64)", |
There was a problem hiding this comment.
Lists have a new render format: apache/arrow-rs#8290
| ( | ||
| f.regexp_replace(column("a"), literal("(ell|orl)"), literal("-")), | ||
| pa.array(["H-o", "W-d", "!"]), | ||
| pa.array(["H-o", "W-d", "!"], type=pa.string_view()), |
There was a problem hiding this comment.
regexp_replace now uses UTF8View: apache/datafusion#17195
| impl PyCatalog { | ||
| #[new] | ||
| fn new(catalog: PyObject) -> Self { | ||
| fn new(catalog: Py<PyAny>) -> Self { |
There was a problem hiding this comment.
PyObject has been deprecated: https://pyo3.rs/main/migration.html#deprecation-of-pyobject-type-alias
| )))?; | ||
|
|
||
| Python::with_gil(|py| { | ||
| Python::attach(|py| { |
There was a problem hiding this comment.
| let streams = spawn_future(py, async move { df.execute_stream_partitioned().await })?; | ||
|
|
||
| let mut schema: Schema = self.df.schema().to_owned().into(); | ||
| let mut schema: Schema = self.df.schema().to_owned().as_arrow().clone(); |
There was a problem hiding this comment.
| #[allow(clippy::too_many_arguments)] | ||
| #[new] | ||
| #[pyo3(signature = (schema, name, location, file_type, table_partition_cols, if_not_exists, temporary, order_exprs, unbounded, options, constraints, column_defaults, definition=None))] | ||
| #[pyo3(signature = (schema, name, location, file_type, table_partition_cols, if_not_exists, or_replace, temporary, order_exprs, unbounded, options, constraints, column_defaults, definition=None))] |
There was a problem hiding this comment.
| impl PyPrepare { | ||
| #[new] | ||
| pub fn new(name: String, data_types: Vec<PyDataType>, input: PyLogicalPlan) -> Self { | ||
| pub fn new(name: String, fields: Vec<PyArrowType<Field>>, input: PyLogicalPlan) -> Self { |
There was a problem hiding this comment.
Prepare now stores fields: apache/datafusion#17986
| use datafusion::functions_aggregate::all_default_aggregate_functions; | ||
| use datafusion::functions_window::all_default_window_functions; | ||
| use datafusion::logical_expr::expr::{Alias, FieldMetadata, WindowFunction, WindowFunctionParams}; | ||
| use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; |
There was a problem hiding this comment.
NullTreatment has been moved to not rely on the sqlparser: apache/datafusion#17332
|
Thank you for getting started on this! I was hoping we might delay slightly until DF 51.1.0 releases apache/datafusion#18843 Do you think it's okay to wait a bit before release? Either way we can get this PR in and the update the lock. |
Yeah no problem! Meanwhile it appears that the actions are stuck on the |
|
I don't know what might be causing the |
|
From a little investigation I suspect that the I wonder if the new |
|
@nuno-faria |
3 participants
Which issue does this PR close?
N/A.
Rationale for this change
Upgrade to the latest version of
datafusion.What changes are included in this PR?
Fixed breaking changes.
Are there any user-facing changes?
No.