Skip to content

Commit

Permalink
Unicode key hash map fix (#670)
Browse files Browse the repository at this point in the history 
* unicode keys testing

* unicode tests and CSV structural reading test

* Adding layout to write_csv

* Hash map fix for unicode
  • Loading branch information
@stephenberry
stephenberry committed Jan 3, 2024
1 parent dcb422a commit d7150f5
Show file tree
Hide file tree
Showing 4 changed files with 242 additions and 6 deletions.
8 changes: 4 additions & 4 deletions include/glaze/csv/write.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,17 +352,17 @@ namespace glz
};
}

template <class T, class Buffer>
template <uint32_t layout = rowwise, class T, class Buffer>
GLZ_ALWAYS_INLINE auto write_csv(T&& value, Buffer&& buffer) noexcept
{
return write<opts{.format = csv}>(std::forward<T>(value), std::forward<Buffer>(buffer));
return write<opts{.format = csv, .layout = layout}>(std::forward<T>(value), std::forward<Buffer>(buffer));
}

template <class T>
template <uint32_t layout = rowwise, class T>
GLZ_ALWAYS_INLINE auto write_csv(T&& value) noexcept
{
std::string buffer{};
write<opts{.format = csv}>(std::forward<T>(value), buffer);
write<opts{.format = csv, .layout = layout}>(std::forward<T>(value), buffer);
return buffer;
}

Expand Down
4 changes: 2 additions & 2 deletions include/glaze/util/hash_map.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ namespace glz::detail
uint64_t res{};
if (std::is_constant_evaluated()) {
for (size_t i = 0; i < N; ++i) {
res |= (uint64_t(bytes[i]) << (i << 3));
res |= (uint64_t(uint8_t(bytes[i])) << (i << 3));
}
}
else {
Expand Down Expand Up @@ -88,7 +88,7 @@ namespace glz::detail
if (std::is_constant_evaluated()) {
uint64_t res{};
for (size_t i = 0; i < N; ++i) {
res |= (uint64_t(bytes[i]) << (i << 3));
res |= (uint64_t(uint8_t(bytes[i])) << (i << 3));
}
return res;
}
Expand Down
143 changes: 143 additions & 0 deletions tests/csv_test/csv_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,4 +348,147 @@ v3s[2],1,2,3,4)");
};
};

struct unicode_keys
{
std::vector<int> field1{0, 1, 2};
std::vector<int> field2{0, 1, 2};
std::vector<int> field3{0, 1, 2};
std::vector<int> field4{0, 1, 2};
std::vector<int> field5{0, 1, 2};
std::vector<int> field6{0, 1, 2};
std::vector<int> field7{0, 1, 2};
};

// example 1
template <>
struct glz::meta<unicode_keys>
{
using T = unicode_keys;
static constexpr auto value = object("alpha",&T::field1,
"bravo",&T::field2,
"charlie",&T::field3,
"♥️",&T::field4,
"delta",&T::field5,
"echo",&T::field6,
"😄", &T::field7
);
};

struct unicode_keys2
{
std::vector<int> field1{0, 1, 2};
std::vector<int> field2{0, 1, 2};
std::vector<int> field3{0, 1, 2};
};

template <>
struct glz::meta<unicode_keys2>
{
using T = unicode_keys2;
static constexpr auto value = object("😄",&T::field1,
"💔",&T::field2,
"alpha",&T::field3
);
};

struct unicode_keys3
{
std::vector<int> field0{0, 1, 2};
std::vector<int> field1{0, 1, 2};
std::vector<int> field2{0, 1, 2};
std::vector<int> field3{0, 1, 2};
std::vector<int> field4{0, 1, 2};
std::vector<int> field5{0, 1, 2};
std::vector<int> field6{0, 1, 2};
};

template <>
struct glz::meta<unicode_keys3>
{
using T = unicode_keys3;
static constexpr auto value = object("简体汉字", &T::field0, // simplified chinese characters
"漢字寿限無寿限無五劫", &T::field1, // traditional chinese characters / kanji
"こんにちはむところやぶら", &T::field2, // katakana
"한국인", &T::field3, // korean
"русский", &T::field4, // cyrillic
"สวัสดี", &T::field5, // thai
"english", &T::field6
);
};

suite unicode_keys_test = [] {
"unicode_keys"_test = [] {
unicode_keys obj{};
std::string buffer{};
glz::write_csv(obj, buffer);

expect(!glz::read_csv(obj, buffer));
};

"unicode_keys2"_test = [] {
unicode_keys2 obj{};
std::string buffer{};
glz::write_csv(obj, buffer);

expect(!glz::read_csv(obj, buffer));
};

"unicode_keys3"_test = [] {
unicode_keys3 obj{};
std::string buffer{};
glz::write_csv(obj, buffer);

expect(!glz::read_csv(obj, buffer));
};
};

struct FishRecord
{
std::vector<float> Duration;
std::vector<float> FishSize;
std::vector<std::uint8_t> Amount;

std::vector<std::string> FishBaitName;
std::vector<std::string> SurfaceSlapFishName;
std::vector<std::string> MoochFishName;
std::vector<std::string> BuffName;
std::vector<std::string> FishingSpotPlaceName;

std::vector<std::string> BiteTypeName;
std::vector<std::string> CaughtFishName;
std::vector<std::string> HooksetName;
std::vector<std::string> IsLargeSizeName;
std::vector<std::string> IsCollectableName;
};

template <>
struct glz::meta<FishRecord>
{
using T = FishRecord;
static constexpr auto value = object("上钩的鱼", &T::CaughtFishName, //
"间隔", &T::Duration, //
"尺寸", &T::FishSize, //
"数量", &T::Amount, //
"鱼饵", &T::FishBaitName, //
"拍水的鱼", &T::SurfaceSlapFishName, //
"以小钓大的鱼", &T::MoochFishName, //
"Buff", &T::BuffName, //
"钓场", &T::FishingSpotPlaceName, //
"咬钩类型", &T::BiteTypeName, //
"提钩类型", &T::HooksetName, //
"大尺寸", &T::IsLargeSizeName, //
"收藏品", &T::IsCollectableName //
);
};

suite fish_record = [] {
"fish_record"_test = [] {
FishRecord obj{};
std::string buffer{};
glz::write_csv<glz::colwise>(obj, buffer);

expect(!glz::read_csv<glz::colwise>(obj, buffer));
};
};

int main() { return boost::ut::cfg<>.run({.report_errors = true}); }
93 changes: 93 additions & 0 deletions tests/json_test/json_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6516,6 +6516,99 @@ suite enum_in_object_reflection_test = [] {
};
};

struct unicode_keys
{
float field1;
float field2;
std::uint8_t field3;
std::string field4;
std::string field5;
std::string field6;
std::string field7;
};

template <>
struct glz::meta<unicode_keys>
{
using T = unicode_keys;
static constexpr auto value = object("alpha",&T::field1,
"bravo",&T::field2,
"charlie",&T::field3,
"♥️",&T::field4,
"delta",&T::field5,
"echo",&T::field6,
"😄",&T::field7
);
};

struct unicode_keys2
{
float field1;
float field2;
std::uint8_t field3;
};

template <>
struct glz::meta<unicode_keys2>
{
using T = unicode_keys2;
static constexpr auto value = object("😄",&T::field1,
"💔",&T::field2,
"alpha",&T::field3
);
};

struct unicode_keys3
{
float field0;
float field1;
float field2;
std::uint8_t field3;
std::string field4;
std::string field5;
std::string field6;
};

template <>
struct glz::meta<unicode_keys3>
{
using T = unicode_keys3;
static constexpr auto value = object("简体汉字", &T::field0, // simplified chinese characters
"漢字寿限無寿限無五劫", &T::field1, // traditional chinese characters / kanji
"こんにちはむところやぶら", &T::field2, // katakana
"한국인", &T::field3, // korean
"русский", &T::field4, // cyrillic
"สวัสดี", &T::field5, // thai
"english", &T::field6
);
};

suite unicode_keys_test = [] {
"unicode_keys"_test = [] {
unicode_keys obj{};
std::string buffer{};
glz::write_json(obj, buffer);

expect(!glz::read_json(obj, buffer));
};

"unicode_keys2"_test = [] {
unicode_keys2 obj{};
std::string buffer{};
glz::write_json(obj, buffer);

expect(!glz::read_json(obj, buffer));
};

"unicode_keys3"_test = [] {
unicode_keys3 obj{};
std::string buffer{};
glz::write_json(obj, buffer);

expect(!glz::read_json(obj, buffer));
};
};

int main()
{
// Explicitly run registered test suites and report errors
Expand Down

0 comments on commit d7150f5

Please sign in to comment.