Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Unicode key hash map fix #670

Merged
merged 5 commits into from
Jan 3, 2024
Merged

Unicode key hash map fix #670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9d99843
unicode keys testing
stephenberry Jan 2, 2024
b37b4c0
unicode tests and CSV structural reading test
stephenberry Jan 2, 2024
5acdaeb
Adding layout to write_csv
stephenberry Jan 2, 2024
1cdb74b
Hash map fix for unicode
stephenberry Jan 3, 2024
f659dd4
Merge branch 'main' into unicode_keys
stephenberry Jan 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions include/glaze/csv/write.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,17 +352,17 @@ namespace glz
};
}

template <class T, class Buffer>
template <uint32_t layout = rowwise, class T, class Buffer>
GLZ_ALWAYS_INLINE auto write_csv(T&& value, Buffer&& buffer) noexcept
{
return write<opts{.format = csv}>(std::forward<T>(value), std::forward<Buffer>(buffer));
return write<opts{.format = csv, .layout = layout}>(std::forward<T>(value), std::forward<Buffer>(buffer));
}

template <class T>
template <uint32_t layout = rowwise, class T>
GLZ_ALWAYS_INLINE auto write_csv(T&& value) noexcept
{
std::string buffer{};
write<opts{.format = csv}>(std::forward<T>(value), buffer);
write<opts{.format = csv, .layout = layout}>(std::forward<T>(value), buffer);
return buffer;
}

Expand Down
4 changes: 2 additions & 2 deletions include/glaze/util/hash_map.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ namespace glz::detail
uint64_t res{};
if (std::is_constant_evaluated()) {
for (size_t i = 0; i < N; ++i) {
res |= (uint64_t(bytes[i]) << (i << 3));
res |= (uint64_t(uint8_t(bytes[i])) << (i << 3));
}
}
else {
Expand Down Expand Up @@ -88,7 +88,7 @@ namespace glz::detail
if (std::is_constant_evaluated()) {
uint64_t res{};
for (size_t i = 0; i < N; ++i) {
res |= (uint64_t(bytes[i]) << (i << 3));
res |= (uint64_t(uint8_t(bytes[i])) << (i << 3));
}
return res;
}
Expand Down
143 changes: 143 additions & 0 deletions tests/csv_test/csv_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,4 +348,147 @@ v3s[2],1,2,3,4)");
};
};

struct unicode_keys
{
std::vector<int> field1{0, 1, 2};
std::vector<int> field2{0, 1, 2};
std::vector<int> field3{0, 1, 2};
std::vector<int> field4{0, 1, 2};
std::vector<int> field5{0, 1, 2};
std::vector<int> field6{0, 1, 2};
std::vector<int> field7{0, 1, 2};
};

// example 1
template <>
struct glz::meta<unicode_keys>
{
using T = unicode_keys;
static constexpr auto value = object("alpha",&T::field1,
"bravo",&T::field2,
"charlie",&T::field3,
"♥️",&T::field4,
"delta",&T::field5,
"echo",&T::field6,
"😄", &T::field7
);
};

struct unicode_keys2
{
std::vector<int> field1{0, 1, 2};
std::vector<int> field2{0, 1, 2};
std::vector<int> field3{0, 1, 2};
};

template <>
struct glz::meta<unicode_keys2>
{
using T = unicode_keys2;
static constexpr auto value = object("😄",&T::field1,
"💔",&T::field2,
"alpha",&T::field3
);
};

struct unicode_keys3
{
std::vector<int> field0{0, 1, 2};
std::vector<int> field1{0, 1, 2};
std::vector<int> field2{0, 1, 2};
std::vector<int> field3{0, 1, 2};
std::vector<int> field4{0, 1, 2};
std::vector<int> field5{0, 1, 2};
std::vector<int> field6{0, 1, 2};
};

template <>
struct glz::meta<unicode_keys3>
{
using T = unicode_keys3;
static constexpr auto value = object("简体汉字", &T::field0, // simplified chinese characters
"漢字寿限無寿限無五劫", &T::field1, // traditional chinese characters / kanji
"こんにちはむところやぶら", &T::field2, // katakana
"한국인", &T::field3, // korean
"русский", &T::field4, // cyrillic
"สวัสดี", &T::field5, // thai
"english", &T::field6
);
};

suite unicode_keys_test = [] {
"unicode_keys"_test = [] {
unicode_keys obj{};
std::string buffer{};
glz::write_csv(obj, buffer);

expect(!glz::read_csv(obj, buffer));
};

"unicode_keys2"_test = [] {
unicode_keys2 obj{};
std::string buffer{};
glz::write_csv(obj, buffer);

expect(!glz::read_csv(obj, buffer));
};

"unicode_keys3"_test = [] {
unicode_keys3 obj{};
std::string buffer{};
glz::write_csv(obj, buffer);

expect(!glz::read_csv(obj, buffer));
};
};

struct FishRecord
{
std::vector<float> Duration;
std::vector<float> FishSize;
std::vector<std::uint8_t> Amount;

std::vector<std::string> FishBaitName;
std::vector<std::string> SurfaceSlapFishName;
std::vector<std::string> MoochFishName;
std::vector<std::string> BuffName;
std::vector<std::string> FishingSpotPlaceName;

std::vector<std::string> BiteTypeName;
std::vector<std::string> CaughtFishName;
std::vector<std::string> HooksetName;
std::vector<std::string> IsLargeSizeName;
std::vector<std::string> IsCollectableName;
};

template <>
struct glz::meta<FishRecord>
{
using T = FishRecord;
static constexpr auto value = object("上钩的鱼", &T::CaughtFishName, //
"间隔", &T::Duration, //
"尺寸", &T::FishSize, //
"数量", &T::Amount, //
"鱼饵", &T::FishBaitName, //
"拍水的鱼", &T::SurfaceSlapFishName, //
"以小钓大的鱼", &T::MoochFishName, //
"Buff", &T::BuffName, //
"钓场", &T::FishingSpotPlaceName, //
"咬钩类型", &T::BiteTypeName, //
"提钩类型", &T::HooksetName, //
"大尺寸", &T::IsLargeSizeName, //
"收藏品", &T::IsCollectableName //
);
};

suite fish_record = [] {
"fish_record"_test = [] {
FishRecord obj{};
std::string buffer{};
glz::write_csv<glz::colwise>(obj, buffer);

expect(!glz::read_csv<glz::colwise>(obj, buffer));
};
};

int main() { return boost::ut::cfg<>.run({.report_errors = true}); }
93 changes: 93 additions & 0 deletions tests/json_test/json_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6516,6 +6516,99 @@ suite enum_in_object_reflection_test = [] {
};
};

struct unicode_keys
{
float field1;
float field2;
std::uint8_t field3;
std::string field4;
std::string field5;
std::string field6;
std::string field7;
};

template <>
struct glz::meta<unicode_keys>
{
using T = unicode_keys;
static constexpr auto value = object("alpha",&T::field1,
"bravo",&T::field2,
"charlie",&T::field3,
"♥️",&T::field4,
"delta",&T::field5,
"echo",&T::field6,
"😄",&T::field7
);
};

struct unicode_keys2
{
float field1;
float field2;
std::uint8_t field3;
};

template <>
struct glz::meta<unicode_keys2>
{
using T = unicode_keys2;
static constexpr auto value = object("😄",&T::field1,
"💔",&T::field2,
"alpha",&T::field3
);
};

struct unicode_keys3
{
float field0;
float field1;
float field2;
std::uint8_t field3;
std::string field4;
std::string field5;
std::string field6;
};

template <>
struct glz::meta<unicode_keys3>
{
using T = unicode_keys3;
static constexpr auto value = object("简体汉字", &T::field0, // simplified chinese characters
"漢字寿限無寿限無五劫", &T::field1, // traditional chinese characters / kanji
"こんにちはむところやぶら", &T::field2, // katakana
"한국인", &T::field3, // korean
"русский", &T::field4, // cyrillic
"สวัสดี", &T::field5, // thai
"english", &T::field6
);
};

suite unicode_keys_test = [] {
"unicode_keys"_test = [] {
unicode_keys obj{};
std::string buffer{};
glz::write_json(obj, buffer);

expect(!glz::read_json(obj, buffer));
};

"unicode_keys2"_test = [] {
unicode_keys2 obj{};
std::string buffer{};
glz::write_json(obj, buffer);

expect(!glz::read_json(obj, buffer));
};

"unicode_keys3"_test = [] {
unicode_keys3 obj{};
std::string buffer{};
glz::write_json(obj, buffer);

expect(!glz::read_json(obj, buffer));
};
};

int main()
{
// Explicitly run registered test suites and report errors
Expand Down