diff --git a/src/odr/internal/csv/csv_document.cpp b/src/odr/internal/csv/csv_document.cpp index 031e52da..f64c0313 100644 --- a/src/odr/internal/csv/csv_document.cpp +++ b/src/odr/internal/csv/csv_document.cpp @@ -220,9 +220,9 @@ class ElementAdapter final : public abstract::ElementAdapter, [[maybe_unused]] const ElementIdentifier element_id) const override { return {1, 1}; } - [[nodiscard]] ValueType sheet_cell_value_type( - [[maybe_unused]] const ElementIdentifier element_id) const override { - return ValueType::string; + [[nodiscard]] ValueType + sheet_cell_value_type(const ElementIdentifier element_id) const override { + return m_document->value_type(column_of(element_id), row_of(element_id)); } // TextAdapter @@ -277,6 +277,31 @@ CsvDocument::CsvDocument(const abstract::File &file, m_dimensions = {static_cast(m_rows.size()), columns}; + // Walks the fields that exist rather than the rectangle they span: one wide + // record widens every row, and scanning `rows * columns` synthesized cells + // costs more than the file holds. + // + // The first row is a header, not a value — one word would otherwise make + // every column prose. + std::vector has_value(columns, false); + m_numeric_columns.assign(columns, true); + for (std::size_t row = 1; row < m_rows.size(); ++row) { + const std::vector &fields = m_rows[row]; + for (std::size_t column = 0; column < fields.size(); ++column) { + const std::string &value = fields[column]; + if (value.empty()) { + continue; + } + has_value[column] = true; + if (!is_number(value)) { + m_numeric_columns[column] = false; + } + } + } + for (std::uint32_t column = 0; column < columns; ++column) { + m_numeric_columns[column] = m_numeric_columns[column] && has_value[column]; + } + m_root_element = make_id(Kind::root); m_element_adapter = std::make_unique(*this); } @@ -314,4 +339,14 @@ TableDimensions CsvDocument::dimensions() const noexcept { return m_dimensions; } +ValueType CsvDocument::value_type(const std::uint32_t column, + const std::uint32_t row) const { + if (column >= m_numeric_columns.size() || !m_numeric_columns[column]) { + return ValueType::string; + } + // the header of a numeric column is still a name + return is_number(cell(column, row)) ? ValueType::float_number + : ValueType::string; +} + } // namespace odr::internal::csv diff --git a/src/odr/internal/csv/csv_document.hpp b/src/odr/internal/csv/csv_document.hpp index e878e65e..124fdfaf 100644 --- a/src/odr/internal/csv/csv_document.hpp +++ b/src/odr/internal/csv/csv_document.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -38,9 +39,16 @@ class CsvDocument final : public internal::Document { std::uint32_t row) const; [[nodiscard]] TableDimensions dimensions() const noexcept; + /// Only @ref ValueType::float_number in a column whose values are all + /// numbers — a lone number in a column of prose is not a quantity. + [[nodiscard]] ValueType value_type(std::uint32_t column, + std::uint32_t row) const; + private: std::vector> m_rows; TableDimensions m_dimensions; + /// Per column, whether every value below the first row is a number. + std::vector m_numeric_columns; }; } // namespace odr::internal::csv diff --git a/src/odr/internal/csv/csv_util.cpp b/src/odr/internal/csv/csv_util.cpp index 47b24c8b..548db589 100644 --- a/src/odr/internal/csv/csv_util.cpp +++ b/src/odr/internal/csv/csv_util.cpp @@ -205,6 +205,57 @@ csv::Probe csv::probe(const std::string_view text, const bool complete, return result; } +bool csv::is_number(std::string_view field) noexcept { + constexpr std::string_view blanks = " \t"; + const std::size_t first = field.find_first_not_of(blanks); + if (first == std::string_view::npos) { + return false; + } + field = field.substr(first, field.find_last_not_of(blanks) - first + 1); + + std::size_t i = 0; + const auto digits = [&] { + const std::size_t start = i; + while (i < field.size() && field[i] >= '0' && field[i] <= '9') { + ++i; + } + return i - start; + }; + + if (i < field.size() && (field[i] == '-' || field[i] == '+')) { + ++i; + } + + const std::size_t integer_start = i; + const std::size_t integer_digits = digits(); + if (integer_digits == 0) { + return false; + } + // a leading zero means the digits are an identifier, not a quantity + if (field[integer_start] == '0' && integer_digits > 1) { + return false; + } + + if (i < field.size() && field[i] == '.') { + ++i; + if (digits() == 0) { + return false; + } + } + + if (i < field.size() && (field[i] == 'e' || field[i] == 'E')) { + ++i; + if (i < field.size() && (field[i] == '-' || field[i] == '+')) { + ++i; + } + if (digits() == 0) { + return false; + } + } + + return i == field.size(); +} + csv::Probe csv::probe(const abstract::File &file, const TextEncoding encoding, const char quote) { if (!text_encoding_is_decodable(encoding)) { diff --git a/src/odr/internal/csv/csv_util.hpp b/src/odr/internal/csv/csv_util.hpp index 1eae933a..647fa0b8 100644 --- a/src/odr/internal/csv/csv_util.hpp +++ b/src/odr/internal/csv/csv_util.hpp @@ -65,6 +65,14 @@ struct Probe final { [[nodiscard]] Probe probe(std::string_view text, bool complete, char quote = '"'); +/// Whether @p field is a number: optional sign, digits, an optional fraction +/// and exponent, surrounding blanks. +/// +/// Rejects a leading zero (`007` is an identifier, not a quantity) and any +/// thousands separator (`1,234` means two different numbers depending on where +/// you live, and a csv does not say which). +[[nodiscard]] bool is_number(std::string_view field) noexcept; + /// Reads @p file's opening bytes, decodes them and scores them. Not a csv when /// @p encoding cannot be decoded. [[nodiscard]] Probe probe(const abstract::File &file, TextEncoding encoding, diff --git a/test/src/internal/csv/csv_file_test.cpp b/test/src/internal/csv/csv_file_test.cpp index 59dc213f..f55da88a 100644 --- a/test/src/internal/csv/csv_file_test.cpp +++ b/test/src/internal/csv/csv_file_test.cpp @@ -298,6 +298,93 @@ TEST(CsvDocument, renders_as_a_table) { EXPECT_THAT(out.str(), testing::HasSubstr(">2<")); } +TEST(CsvNumbers, a_number_is_a_number) { + EXPECT_TRUE(csv::is_number("0")); + EXPECT_TRUE(csv::is_number("42")); + EXPECT_TRUE(csv::is_number("-42")); + EXPECT_TRUE(csv::is_number("+42")); + EXPECT_TRUE(csv::is_number("3.14")); + EXPECT_TRUE(csv::is_number("0.5")); + EXPECT_TRUE(csv::is_number("-0.5")); + EXPECT_TRUE(csv::is_number("1e9")); + EXPECT_TRUE(csv::is_number("1.5E-3")); + EXPECT_TRUE(csv::is_number(" 42 ")); + + EXPECT_FALSE(csv::is_number("")); + EXPECT_FALSE(csv::is_number(" ")); + EXPECT_FALSE(csv::is_number("abc")); + EXPECT_FALSE(csv::is_number("42abc")); + EXPECT_FALSE(csv::is_number("4 2")); + EXPECT_FALSE(csv::is_number(".")); + EXPECT_FALSE(csv::is_number("1.")); + EXPECT_FALSE(csv::is_number("1e")); + EXPECT_FALSE(csv::is_number("-")); +} + +/// A leading zero is what tells a code from a quantity, and a thousands +/// separator does not say which side of the Atlantic wrote it. +TEST(CsvNumbers, an_identifier_is_not_a_number) { + EXPECT_FALSE(csv::is_number("007")); + EXPECT_FALSE(csv::is_number("0123456789012")); + EXPECT_FALSE(csv::is_number("1,234")); + EXPECT_FALSE(csv::is_number("1.234,56")); + EXPECT_FALSE(csv::is_number("1,234.56")); + // dates are left alone entirely + EXPECT_FALSE(csv::is_number("2026-08-09")); + EXPECT_FALSE(csv::is_number("03/04/2026")); +} + +namespace { + +ValueType value_type_at(const std::string &content, const std::uint32_t column, + const std::uint32_t row) { + const CsvFile file = CsvFile::from_file(File::from_memory(content), + CsvOptions{.separator = ','}); + const Document document = file.document(); + const Sheet sheet = (*document.root_element().children().begin()).as_sheet(); + return sheet.cell(column, row).value_type(); +} + +} // namespace + +TEST(CsvValueType, a_numeric_column_is_a_number) { + const std::string content = "name,age\nx,42\ny,7\n"; + + EXPECT_EQ(value_type_at(content, 1, 1), ValueType::float_number); + EXPECT_EQ(value_type_at(content, 1, 2), ValueType::float_number); + EXPECT_EQ(value_type_at(content, 0, 1), ValueType::string); + // the header of a numeric column is still a name + EXPECT_EQ(value_type_at(content, 1, 0), ValueType::string); +} + +/// A reader compares down a column, so one number in a column of prose is not +/// a quantity. +TEST(CsvValueType, a_lone_number_in_a_text_column_is_not) { + EXPECT_EQ(value_type_at("a,b\nx,note\ny,42\n", 1, 2), ValueType::string); +} + +TEST(CsvValueType, an_empty_cell_does_not_break_a_numeric_column) { + const std::string content = "a,b\nx,1\ny,\nz,3\n"; + EXPECT_EQ(value_type_at(content, 1, 1), ValueType::float_number); + EXPECT_EQ(value_type_at(content, 1, 2), ValueType::string); + EXPECT_EQ(value_type_at(content, 1, 3), ValueType::float_number); +} + +TEST(CsvValueType, a_column_of_codes_stays_text) { + EXPECT_EQ(value_type_at("a,code\nx,007\ny,008\n", 1, 1), ValueType::string); +} + +/// A column only one record reaches is padding everywhere else, and padding is +/// not a value: the inference sees the one field, not the empty rectangle +/// around it. +TEST(CsvValueType, a_column_one_wide_record_opened_holds_one_value) { + const std::string content = "a\nb\nc,1\nd\n"; + + EXPECT_EQ(value_type_at(content, 1, 2), ValueType::float_number); + EXPECT_EQ(value_type_at(content, 1, 1), ValueType::string); + EXPECT_EQ(value_type_at(content, 1, 3), ValueType::string); +} + /// Cells are not reachable by walking, so the generic path machinery has to /// get at them the other way — through `sheet_cell`. TEST(CsvDocument, a_cell_path_round_trips) {