File tree Expand file tree Collapse file tree 1 file changed +7
-1
lines changed Expand file tree Collapse file tree 1 file changed +7
-1
lines changed Original file line number Diff line number Diff line change 99
99
HDFS_KERB_TICKET ,
100
100
HDFS_PORT ,
101
101
HDFS_USER ,
102
+ PYARROW_USE_LARGE_TYPES_ON_READ ,
102
103
S3_ACCESS_KEY_ID ,
103
104
S3_CONNECT_TIMEOUT ,
104
105
S3_ENDPOINT ,
@@ -1560,11 +1561,16 @@ def _table_from_scan_task(task: FileScanTask) -> pa.Table:
1560
1561
1561
1562
tables = [f .result () for f in completed_futures if f .result ()]
1562
1563
1564
+ arrow_schema = schema_to_pyarrow (self ._projected_schema , include_field_ids = False )
1565
+
1563
1566
if len (tables ) < 1 :
1564
- return pa .Table .from_batches ([], schema = schema_to_pyarrow ( self . _projected_schema , include_field_ids = False ) )
1567
+ return pa .Table .from_batches ([], schema = arrow_schema )
1565
1568
1566
1569
result = pa .concat_tables (tables , promote_options = "permissive" )
1567
1570
1571
+ if property_as_bool (self ._io .properties , PYARROW_USE_LARGE_TYPES_ON_READ , False ):
1572
+ result = result .cast (arrow_schema )
1573
+
1568
1574
if self ._limit is not None :
1569
1575
return result .slice (0 , self ._limit )
1570
1576
You can’t perform that action at this time.
0 commit comments