Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 38 additions & 3 deletions src/odr/internal/csv/csv_document.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -220,9 +220,9 @@ class ElementAdapter final : public abstract::ElementAdapter,
[[maybe_unused]] const ElementIdentifier element_id) const override {
return {1, 1};
}
[[nodiscard]] ValueType sheet_cell_value_type(
[[maybe_unused]] const ElementIdentifier element_id) const override {
return ValueType::string;
[[nodiscard]] ValueType
sheet_cell_value_type(const ElementIdentifier element_id) const override {
return m_document->value_type(column_of(element_id), row_of(element_id));
}

// TextAdapter
Expand Down Expand Up @@ -277,6 +277,31 @@ CsvDocument::CsvDocument(const abstract::File &file,

m_dimensions = {static_cast<std::uint32_t>(m_rows.size()), columns};

// Walks the fields that exist rather than the rectangle they span: one wide
// record widens every row, and scanning `rows * columns` synthesized cells
// costs more than the file holds.
//
// The first row is a header, not a value — one word would otherwise make
// every column prose.
std::vector<bool> has_value(columns, false);
m_numeric_columns.assign(columns, true);
for (std::size_t row = 1; row < m_rows.size(); ++row) {
const std::vector<std::string> &fields = m_rows[row];
for (std::size_t column = 0; column < fields.size(); ++column) {
const std::string &value = fields[column];
if (value.empty()) {
continue;
}
has_value[column] = true;
if (!is_number(value)) {
m_numeric_columns[column] = false;
}
}
}
for (std::uint32_t column = 0; column < columns; ++column) {
m_numeric_columns[column] = m_numeric_columns[column] && has_value[column];
}

m_root_element = make_id(Kind::root);
m_element_adapter = std::make_unique<ElementAdapter>(*this);
}
Expand Down Expand Up @@ -314,4 +339,14 @@ TableDimensions CsvDocument::dimensions() const noexcept {
return m_dimensions;
}

ValueType CsvDocument::value_type(const std::uint32_t column,
const std::uint32_t row) const {
if (column >= m_numeric_columns.size() || !m_numeric_columns[column]) {
return ValueType::string;
}
// the header of a numeric column is still a name
return is_number(cell(column, row)) ? ValueType::float_number
: ValueType::string;
}

} // namespace odr::internal::csv
8 changes: 8 additions & 0 deletions src/odr/internal/csv/csv_document.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#pragma once

#include <odr/document_element.hpp>
#include <odr/file.hpp>
#include <odr/table_dimension.hpp>

Expand Down Expand Up @@ -38,9 +39,16 @@ class CsvDocument final : public internal::Document {
std::uint32_t row) const;
[[nodiscard]] TableDimensions dimensions() const noexcept;

/// Only @ref ValueType::float_number in a column whose values are all
/// numbers — a lone number in a column of prose is not a quantity.
[[nodiscard]] ValueType value_type(std::uint32_t column,
std::uint32_t row) const;

private:
std::vector<std::vector<std::string>> m_rows;
TableDimensions m_dimensions;
/// Per column, whether every value below the first row is a number.
std::vector<bool> m_numeric_columns;
};

} // namespace odr::internal::csv
51 changes: 51 additions & 0 deletions src/odr/internal/csv/csv_util.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,57 @@ csv::Probe csv::probe(const std::string_view text, const bool complete,
return result;
}

bool csv::is_number(std::string_view field) noexcept {
constexpr std::string_view blanks = " \t";
const std::size_t first = field.find_first_not_of(blanks);
if (first == std::string_view::npos) {
return false;
}
field = field.substr(first, field.find_last_not_of(blanks) - first + 1);

std::size_t i = 0;
const auto digits = [&] {
const std::size_t start = i;
while (i < field.size() && field[i] >= '0' && field[i] <= '9') {
++i;
}
return i - start;
};

if (i < field.size() && (field[i] == '-' || field[i] == '+')) {
++i;
}

const std::size_t integer_start = i;
const std::size_t integer_digits = digits();
if (integer_digits == 0) {
return false;
}
// a leading zero means the digits are an identifier, not a quantity
if (field[integer_start] == '0' && integer_digits > 1) {
return false;
}

if (i < field.size() && field[i] == '.') {
++i;
if (digits() == 0) {
return false;
}
}

if (i < field.size() && (field[i] == 'e' || field[i] == 'E')) {
++i;
if (i < field.size() && (field[i] == '-' || field[i] == '+')) {
++i;
}
if (digits() == 0) {
return false;
}
}

return i == field.size();
}

csv::Probe csv::probe(const abstract::File &file, const TextEncoding encoding,
const char quote) {
if (!text_encoding_is_decodable(encoding)) {
Expand Down
8 changes: 8 additions & 0 deletions src/odr/internal/csv/csv_util.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,14 @@ struct Probe final {
[[nodiscard]] Probe probe(std::string_view text, bool complete,
char quote = '"');

/// Whether @p field is a number: optional sign, digits, an optional fraction
/// and exponent, surrounding blanks.
///
/// Rejects a leading zero (`007` is an identifier, not a quantity) and any
/// thousands separator (`1,234` means two different numbers depending on where
/// you live, and a csv does not say which).
[[nodiscard]] bool is_number(std::string_view field) noexcept;

/// Reads @p file's opening bytes, decodes them and scores them. Not a csv when
/// @p encoding cannot be decoded.
[[nodiscard]] Probe probe(const abstract::File &file, TextEncoding encoding,
Expand Down
87 changes: 87 additions & 0 deletions test/src/internal/csv/csv_file_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,93 @@ TEST(CsvDocument, renders_as_a_table) {
EXPECT_THAT(out.str(), testing::HasSubstr(">2<"));
}

TEST(CsvNumbers, a_number_is_a_number) {
EXPECT_TRUE(csv::is_number("0"));
EXPECT_TRUE(csv::is_number("42"));
EXPECT_TRUE(csv::is_number("-42"));
EXPECT_TRUE(csv::is_number("+42"));
EXPECT_TRUE(csv::is_number("3.14"));
EXPECT_TRUE(csv::is_number("0.5"));
EXPECT_TRUE(csv::is_number("-0.5"));
EXPECT_TRUE(csv::is_number("1e9"));
EXPECT_TRUE(csv::is_number("1.5E-3"));
EXPECT_TRUE(csv::is_number(" 42 "));

EXPECT_FALSE(csv::is_number(""));
EXPECT_FALSE(csv::is_number(" "));
EXPECT_FALSE(csv::is_number("abc"));
EXPECT_FALSE(csv::is_number("42abc"));
EXPECT_FALSE(csv::is_number("4 2"));
EXPECT_FALSE(csv::is_number("."));
EXPECT_FALSE(csv::is_number("1."));
EXPECT_FALSE(csv::is_number("1e"));
EXPECT_FALSE(csv::is_number("-"));
}

/// A leading zero is what tells a code from a quantity, and a thousands
/// separator does not say which side of the Atlantic wrote it.
TEST(CsvNumbers, an_identifier_is_not_a_number) {
EXPECT_FALSE(csv::is_number("007"));
EXPECT_FALSE(csv::is_number("0123456789012"));
EXPECT_FALSE(csv::is_number("1,234"));
EXPECT_FALSE(csv::is_number("1.234,56"));
EXPECT_FALSE(csv::is_number("1,234.56"));
// dates are left alone entirely
EXPECT_FALSE(csv::is_number("2026-08-09"));
EXPECT_FALSE(csv::is_number("03/04/2026"));
}

namespace {

ValueType value_type_at(const std::string &content, const std::uint32_t column,
const std::uint32_t row) {
const CsvFile file = CsvFile::from_file(File::from_memory(content),
CsvOptions{.separator = ','});
const Document document = file.document();
const Sheet sheet = (*document.root_element().children().begin()).as_sheet();
return sheet.cell(column, row).value_type();
}

} // namespace

TEST(CsvValueType, a_numeric_column_is_a_number) {
const std::string content = "name,age\nx,42\ny,7\n";

EXPECT_EQ(value_type_at(content, 1, 1), ValueType::float_number);
EXPECT_EQ(value_type_at(content, 1, 2), ValueType::float_number);
EXPECT_EQ(value_type_at(content, 0, 1), ValueType::string);
// the header of a numeric column is still a name
EXPECT_EQ(value_type_at(content, 1, 0), ValueType::string);
}

/// A reader compares down a column, so one number in a column of prose is not
/// a quantity.
TEST(CsvValueType, a_lone_number_in_a_text_column_is_not) {
EXPECT_EQ(value_type_at("a,b\nx,note\ny,42\n", 1, 2), ValueType::string);
}

TEST(CsvValueType, an_empty_cell_does_not_break_a_numeric_column) {
const std::string content = "a,b\nx,1\ny,\nz,3\n";
EXPECT_EQ(value_type_at(content, 1, 1), ValueType::float_number);
EXPECT_EQ(value_type_at(content, 1, 2), ValueType::string);
EXPECT_EQ(value_type_at(content, 1, 3), ValueType::float_number);
}

TEST(CsvValueType, a_column_of_codes_stays_text) {
EXPECT_EQ(value_type_at("a,code\nx,007\ny,008\n", 1, 1), ValueType::string);
}

/// A column only one record reaches is padding everywhere else, and padding is
/// not a value: the inference sees the one field, not the empty rectangle
/// around it.
TEST(CsvValueType, a_column_one_wide_record_opened_holds_one_value) {
const std::string content = "a\nb\nc,1\nd\n";

EXPECT_EQ(value_type_at(content, 1, 2), ValueType::float_number);
EXPECT_EQ(value_type_at(content, 1, 1), ValueType::string);
EXPECT_EQ(value_type_at(content, 1, 3), ValueType::string);
}

/// Cells are not reachable by walking, so the generic path machinery has to
/// get at them the other way — through `sheet_cell`.
TEST(CsvDocument, a_cell_path_round_trips) {
Expand Down
Loading