csv-parsing automatically parse dates

polars/Cargo.toml

@@ -13,7 +13,7 @@ rows = ["polars-core/rows"]
 simd = ["polars-core/simd"]
 avx512 = ["polars-core/avx512"]
 docs = ["polars-core/docs"]
-temporal = ["polars-core/temporal", "polars-lazy/temporal"]
+temporal = ["polars-core/temporal", "polars-lazy/temporal", "polars-io/temporal"]
 random = ["polars-core/random"]
 default = ["docs",
     "zip_with",

polars/polars-core/src/frame/mod.rs

@@ -1812,6 +1812,12 @@ impl Default for DataFrame {
     }
 }
 
+impl From<DataFrame> for Vec<Series> {
+    fn from(df: DataFrame) -> Self {
+        df.columns
+    }
+}
+
 /// Conversion from Vec<RecordBatch> into DataFrame
 ///
 ///

polars/polars-io/Cargo.toml

@@ -24,6 +24,7 @@ csv-file = ["csv-core", "memmap", "lexical", "arrow/io_csv"]
 fmt = ["polars-core/plain_fmt"]
 decompress = ["flate2/miniz_oxide"]
 decompress-fast = ["flate2/zlib-ng-compat"]
+temporal = ["polars-core/dtype-date32", "polars-core/dtype-date64"]
 # don't use this
 private = []
 

polars/polars-io/src/csv.rs

@@ -214,6 +214,8 @@ where
     null_values: Option<NullValues>,
     predicate: Option<Arc<dyn PhysicalIoExpr>>,
     aggregate: Option<&'a [ScanAggregation]>,
+    #[cfg(feature = "temporal")]
+    parse_dates: bool,
 }
 
 impl<'a, R> CsvReader<'a, R>
@@ -359,6 +361,13 @@ where
         self
     }
 
+    /// Automatically try to parse dates/ datetimes. If parsing failes, columns remain of dtype Utf8.
+    #[cfg(feature = "temporal")]
+    pub fn with_parse_dates(mut self, toggle: bool) -> Self {
+        self.parse_dates = toggle;
+        self
+    }
+
     #[cfg(feature = "private")]
     pub fn with_predicate(mut self, predicate: Option<Arc<dyn PhysicalIoExpr>>) -> Self {
         self.predicate = predicate;
@@ -410,6 +419,8 @@ where
             null_values: None,
             predicate: None,
             aggregate: None,
+            #[cfg(feature = "temporal")]
+            parse_dates: false,
         }
     }
 
@@ -510,10 +521,32 @@ where
         if rechunk && df.n_chunks()? > 1 {
             df.as_single_chunk();
         }
+        #[cfg(feature = "temporal")]
+        if self.parse_dates {
+            df = parse_dates(df)
+        }
         Ok(df)
     }
 }
 
+#[cfg(feature = "temporal")]
+fn parse_dates(df: DataFrame) -> DataFrame {
+    let mut cols: Vec<Series> = df.into();
+
+    for s in cols.iter_mut() {
+        if let Ok(ca) = s.utf8() {
+            // the order is important. A datetime can always be parsed as date.
+            if let Ok(ca) = ca.as_date64(None) {
+                *s = ca.into_series()
+            } else if let Ok(ca) = ca.as_date32(None) {
+                *s = ca.into_series()
+            }
+        }
+    }
+
+    DataFrame::new_no_checks(cols)
+}
+
 #[cfg(test)]
 mod test {
     use crate::prelude::*;
@@ -1043,4 +1076,23 @@ bar,bar";
         assert!(df.frame_equal(&expect));
         Ok(())
     }
+
+    #[test]
+    fn test_automatic_datetime_parsing() -> Result<()> {
+        let csv = r"timestamp,open,high
+2021-01-01 00:00:00,0.00305500,0.00306000
+2021-01-01 00:15:00,0.00298800,0.00300400
+2021-01-01 00:30:00,0.00298300,0.00300100
+2021-01-01 00:45:00,0.00299400,0.00304000
+";
+
+        let file = Cursor::new(csv);
+        let df = CsvReader::new(file).with_parse_dates(true).finish()?;
+
+        let ts = df.column("timestamp")?;
+        assert_eq!(ts.dtype(), &DataType::Date64);
+        assert_eq!(ts.null_count(), 0);
+
+        Ok(())
+    }
 }

py-polars/polars/eager/frame.py

@@ -454,6 +454,7 @@ def read_csv(
         low_memory: bool = False,
         comment_char: Optional[str] = None,
         null_values: Optional[Union[str, tp.List[str], Dict[str, str]]] = None,
+        parse_dates: bool = True,
     ) -> "DataFrame":
         """
         Read a CSV file into a Dataframe.
@@ -558,6 +559,7 @@ def read_csv(
             low_memory,
             comment_char,
             processed_null_values,
+            parse_dates,
         )
         return self
 

py-polars/polars/io.py

@@ -159,6 +159,7 @@ def read_csv(
     comment_char: Optional[str] = None,
     storage_options: Optional[Dict] = None,
     null_values: Optional[Union[str, List[str], Dict[str, str]]] = None,
+    parse_dates: bool = True,
 ) -> "pl.DataFrame":
     """
     Read into a DataFrame from a csv file.
@@ -220,6 +221,9 @@ def read_csv(
         - str -> all values encountered equal to this string will be null
         - List[str] -> A null value per column.
         - Dict[str, str] -> A dictionary that maps column name to a null value string.
+    parse_dates
+        Try to automatically parse dates. If this not succeeds, the column remains
+        of data type Utf8.
 
     Returns
     -------
@@ -246,6 +250,7 @@ def read_csv(
         and encoding == "utf8"
         and not low_memory
         and null_values is None
+        and parse_dates
     ):
         include_columns = None
 
@@ -305,6 +310,7 @@ def read_csv(
             low_memory=low_memory,
             comment_char=comment_char,
             null_values=null_values,
+            parse_dates=parse_dates,
         )
 
     if new_columns:

py-polars/src/dataframe.rs

@@ -98,6 +98,7 @@ impl PyDataFrame {
         low_memory: bool,
         comment_char: Option<&str>,
         null_values: Option<Wrap<NullValues>>,
+        parse_dates: bool
     ) -> PyResult<Self> {
         let null_values = null_values.map(|w| w.0);
         let comment_char = comment_char.map(|s| s.as_bytes()[0]);
@@ -153,6 +154,7 @@ impl PyDataFrame {
             .low_memory(low_memory)
             .with_comment_char(comment_char)
             .with_null_values(null_values)
+            .with_parse_dates(parse_dates)
             .finish()
             .map_err(PyPolarsEr::from)?;
         Ok(df.into())

py-polars/tests/test_io.py

@@ -114,6 +114,20 @@ def test_csv_null_values():
     assert df[1, "b"] is None
 
 
+def test_datetime_parsing():
+    csv = """
+timestamp,open,high
+2021-01-01 00:00:00,0.00305500,0.00306000
+2021-01-01 00:15:00,0.00298800,0.00300400
+2021-01-01 00:30:00,0.00298300,0.00300100
+2021-01-01 00:45:00,0.00299400,0.00304000
+"""
+
+    f = io.StringIO(csv)
+    df = pl.read_csv(f)
+    assert df.dtypes == [pl.Date64, pl.Float64, pl.Float64]
+
+
 def test_partial_dtype_overwrite():
     csv = """
 a,b,c