add decimal_text_to_i128

Changelog.md

@@ -1,8 +1,9 @@
 # Changelog
 
-## (next)
+## 5.0.0
 
-* Removed reexport of force_send_sync
+* removed reexport of `force_send_sync`. It is no longer used within `odbc-api` this should have been removed together with the methods which were promoting Connections to `Send`, but has been overlooked.
+* Adds `decimal_text_to_i128` a useful function for downstream applications including `odbc2parquet` and `arrow-odbc`.
 
 ## 4.1.0
 

odbc-api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "odbc-api"
-version = "4.1.0"
+version = "5.0.0"
 authors = ["Markus Klein"]
 edition = "2021"
 license = "MIT"
@@ -72,6 +72,7 @@ thiserror = "1.0.56"
 log = "0.4.20"
 # Interacting with UTF-16 texts for wide columns or wide function calls
 widestring = "1.0.2"
+atoi = "2.0.0"
 
 [target.'cfg(windows)'.dependencies]
 # We use winit to display dialogs prompting for connection strings. We can deactivate default

odbc-api/src/conversion.rs

@@ -0,0 +1,63 @@
+use atoi::{FromRadix10, FromRadix10Signed};
+
+/// Convert the text representation of a decimal into an integer representation. The integer
+/// representation is not truncating the fraction, but is instead the value of the decimal times 10
+/// to the power of scale. E.g. 123.45 of a Decimal with scale 3 is thought of as 123.450 and
+/// represented as 123450. This method will regard any non digit character as a radix character with
+/// the exception of a `+` or `-` at the beginning of the string.
+/// 
+/// This method is robust against representation which do not have trailing zeroes as well as
+/// arbitrary radix character. If you do not write a generic application and now the specific way
+/// your database formats decimals you may come up with faster methods to parse decimals.
+pub fn decimal_text_to_i128(text: &[u8], scale: usize) -> i128 {
+    // High is now the number before the decimal point
+    let (mut high, num_digits_high) = i128::from_radix_10_signed(text);
+    let (low, num_digits_low) = if num_digits_high == text.len() {
+        (0, 0)
+    } else {
+        i128::from_radix_10(&text[(num_digits_high + 1)..])
+    };
+    // Left shift high so it is compatible with low
+    for _ in 0..num_digits_low {
+        high *= 10;
+    }
+    // We want to increase the absolute of high by low without changing highs sign
+    let mut n = if high < 0 { high - low } else { high + low };
+    // We would be done now, if every database would include trailing zeroes, but they might choose
+    // to omit those. Therfore we see if we need to leftshift n further in order to meet scale.
+    for _ in 0..(scale - num_digits_low) {
+        n *= 10;
+    }
+    n
+}
+
+#[cfg(test)]
+mod tests {
+    use super::decimal_text_to_i128;
+
+    /// An user of an Oracle database got invalid values from decimal after setting
+    /// `NLS_NUMERIC_CHARACTERS` to ",." instead of ".".
+    ///
+    /// See issue:
+    /// <https://github.com/pacman82/arrow-odbc-py/discussions/74#discussioncomment-8083928>
+    #[test]
+    fn decimal_is_represented_with_comma_as_radix() {
+        let actual = decimal_text_to_i128(b"10,00000", 5);
+        assert_eq!(1_000_000, actual);
+    }
+
+    /// Since scale is 5 in this test case we would expect five digits after the radix, yet Oracle
+    /// seems to not emit trailing zeroes. Also see issue:
+    /// <https://github.com/pacman82/arrow-odbc-py/discussions/74#discussioncomment-8083928>
+    #[test]
+    fn decimal_with_less_zeroes() {
+        let actual = decimal_text_to_i128(b"10.0", 5);
+        assert_eq!(1_000_000, actual);
+    }
+
+    #[test]
+    fn negative_decimal() {
+        let actual = decimal_text_to_i128(b"-10.00000", 5);
+        assert_eq!(-1_000_000, actual);
+    }
+}

odbc-api/src/lib.rs

@@ -6,6 +6,7 @@
 
 mod columnar_bulk_inserter;
 mod connection;
+mod conversion;
 mod cursor;
 mod driver_complete_option;
 mod environment;
@@ -31,6 +32,7 @@ pub mod parameter;
 pub use self::{
     columnar_bulk_inserter::{BoundInputSlice, ColumnarBulkInserter},
     connection::{escape_attribute_value, Connection, ConnectionOptions},
+    conversion::decimal_text_to_i128,
     cursor::{
         BlockCursor, BlockCursorPolling, Cursor, CursorImpl, CursorPolling, CursorRow,
         RowSetBuffer, TruncationInfo,

odbc-api/tests/integration.rs

@@ -18,7 +18,7 @@ use odbc_api::{
     parameter::{InputParameter, VarCharSliceMut},
     sys, Bit, ColumnDescription, Connection, ConnectionOptions, Cursor, DataType, Error, InOut,
     IntoParameter, Narrow, Nullability, Nullable, Out, Preallocated, ResultSetMetadata, U16Str,
-    U16String,
+    U16String, decimal_text_to_i128,
 };
 use std::{
     ffi::CString,
@@ -4045,6 +4045,40 @@ fn json_column_display_size(profile: &Profile, expected_display_size: Option<usi
     assert_eq!(expected_display_size, size);
 }
 
+/// Fetch decimal values in their text interpretation and transform them to their integer
+/// representation.
+#[test_case(MSSQL; "Microsoft SQL Server")]
+#[test_case(MARIADB; "Maria DB")]
+#[test_case(SQLITE_3; "SQLite 3")]
+#[test_case(POSTGRES; "PostgreSQL")]
+fn fetch_decimals_to_int(profile: &Profile) {
+    // Given
+    let table_name = table_name!();
+    let (conn, table) = profile.given(&table_name, &["DECIMAL(5,3)"]).unwrap();
+    conn.execute(
+        &format!("INSERT INTO {table_name} (a) VALUES (12.345), (-12.345), (12), (12.3)"),
+        (),
+    )
+    .unwrap();
+
+    // When
+    let mut cursor = conn.execute(&table.sql_all_ordered_by_id(), ()).unwrap().unwrap();
+    let row_set_buffer = TextRowSet::for_cursor(4, &mut cursor, None).unwrap();
+    let mut block_cursor = cursor.bind_buffer(row_set_buffer).unwrap();
+    let batch = block_cursor.fetch().unwrap().unwrap();
+    let n = |i| batch.at_as_str(0,i).unwrap().unwrap().as_bytes();
+    let n1 = decimal_text_to_i128(n(0), 3);
+    let n2 = decimal_text_to_i128(n(1), 3);
+    let n3 = decimal_text_to_i128(n(2), 3);
+    let n4 = decimal_text_to_i128(n(3), 3);
+
+    // Then
+    assert_eq!(12345, n1);
+    assert_eq!(-12345, n2);
+    assert_eq!(12000, n3);
+    assert_eq!(12300, n4);
+}
+
 #[test_case(MSSQL; "Microsoft SQL Server")]
 #[test_case(MARIADB; "Maria DB")]
 #[test_case(SQLITE_3; "SQLite 3")]

odbcsv/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "odbcsv"
-version = "1.0.3"
+version = "1.0.4"
 authors = ["Markus Klein"]
 edition = "2021"
 license = "MIT"
@@ -29,7 +29,7 @@ readme = "Readme.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-odbc-api = { version = "4.1.0", path = "../odbc-api" }
+odbc-api = { version = "5.0.0", path = "../odbc-api" }
 csv = "1.3.0"
 anyhow = "1.0.79"
 stderrlog = "0.5.4"

odbcsv/Changelog.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 1.0.4
+
+* Updated dependencies
+
 ## 1.0.3
 
 * Improved diagnostics in case of truncation. Error now describes, in which column the truncation occurred.