Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Updating to On Demand #1

Closed
wants to merge 27 commits into from
Closed

[WIP] Updating to On Demand #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
ee69a36
Update simdjson.cpp and simdjson.h.
NicolasJiaxin Jul 28, 2021
08d06fe
Groundwork.
NicolasJiaxin Jul 29, 2021
f6c2fd7
Typos.
NicolasJiaxin Jul 29, 2021
a4d2bd5
Update matrix deserialization.
NicolasJiaxin Jul 29, 2021
1e56b7c
Update data frame serialization
NicolasJiaxin Jul 29, 2021
7f416ba
Treating numbers as doubles.
NicolasJiaxin Jul 30, 2021
17bc530
Vectors.
NicolasJiaxin Jul 30, 2021
be99bcd
Matrices.
NicolasJiaxin Aug 2, 2021
aa237ee
Data frames.
NicolasJiaxin Aug 2, 2021
229c7ab
Parsing & loading.
NicolasJiaxin Aug 2, 2021
c915154
Some typos.
NicolasJiaxin Aug 2, 2021
0c84726
Fixes.
NicolasJiaxin Aug 2, 2021
6829926
Fix.
NicolasJiaxin Aug 2, 2021
cb1cea8
Making get_scalar const.
NicolasJiaxin Aug 2, 2021
83e60ce
Fix.
NicolasJiaxin Aug 2, 2021
1fbe6e6
Fixes.
NicolasJiaxin Aug 3, 2021
f151011
Casting issues.
NicolasJiaxin Aug 3, 2021
98926af
Typo.
NicolasJiaxin Aug 3, 2021
fc19952
Fix.
NicolasJiaxin Aug 3, 2021
b2cac33
Small fix
NicolasJiaxin Aug 3, 2021
462e768
Attempt to fix qualifiers issue.
NicolasJiaxin Aug 3, 2021
3465c7d
Attempt
NicolasJiaxin Aug 3, 2021
81c3887
Fix qualifiers issue.
NicolasJiaxin Aug 4, 2021
ea8d829
Fixes.
NicolasJiaxin Aug 4, 2021
c09254e
Type fixes.
NicolasJiaxin Aug 4, 2021
16e01ba
Typos.
NicolasJiaxin Aug 4, 2021
47d9fec
Maybe final fixes.
NicolasJiaxin Aug 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions inst/include/RcppSimdJson/common.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ static inline constexpr int64_t NA_INTEGER64 = LLONG_MIN;


/**
* @brief Typing arguments that decide how a simdjson::dom::element is ultimately returned to R.
* @brief Typing arguments that decide how a simdjson::ondemand::value is ultimately returned to R.
*/
enum class rcpp_T : int {
array = 0, /**< recursive: individual elements will decide ultimate R type */
Expand Down Expand Up @@ -151,12 +151,12 @@ namespace deserialize {


/**
* @brief Simplify asimdjson::dom::element to an R object.
* @brief Simplify asimdjson::ondemand::value to an R object.
*
* @note Forward declaration. See inst/include/RcppSimdJson/deserialize/simplify.hpp.
*/
template <Type_Policy type_policy, utils::Int64_R_Type int64_opt, Simplify_To simplify_to>
inline auto simplify_element(simdjson::dom::element element,
inline auto simplify_element(simdjson::ondemand::value value,
SEXP empty_array,
SEXP empty_object,
SEXP single_null) -> SEXP;
Expand Down
54 changes: 28 additions & 26 deletions inst/include/RcppSimdJson/deserialize.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ struct Parse_Opts {


/**
* @brief Deserialize a parsed simdjson::dom::element to R objects.
* @brief Deserialize a parsed simdjson::ondemand::value to R objects.
*
*
* @param element simdjson::dom::element to deserialize.
* @param element simdjson::ondemand::value to deserialize.
*
* @param empty_array R object to return when encountering an empty JSON array.
*
Expand All @@ -58,7 +58,7 @@ struct Parse_Opts {
*
* @return The simplified R object ( SEXP ).
*/
inline auto deserialize(simdjson::dom::element parsed, const Parse_Opts& parse_opts) -> SEXP {
inline auto deserialize(simdjson::ondemand::value parsed, const Parse_Opts& parse_opts) -> SEXP {
using Int64_R_Type = utils::Int64_R_Type;

auto& [simplify_to, type_policy, int64_opt, empty_array, empty_object, single_null] =
Expand Down Expand Up @@ -422,12 +422,12 @@ inline auto deserialize(simdjson::dom::element parsed, const Parse_Opts& parse_o


template <typename json_T, bool is_file>
inline simdjson::simdjson_result<simdjson::dom::element> parse(simdjson::dom::parser& parser,
inline simdjson::simdjson_result<simdjson::ondemand::document> parse(simdjson::ondemand::parser& parser,
const json_T& json) {
if constexpr (utils::resembles_vec_raw<json_T>()) {
/* if `json` is a raw (unsigned char) vector, we can cheat */
return parser.parse(
std::string_view(reinterpret_cast<const char*>(&(json[0])), std::size(json)));
simdjson::padded_string content = simdjson::padded_string(reinterpret_cast<const char*>( &(json[0]), std::size(json) );
return parser.iterate(content);
}

if constexpr (utils::resembles_vec_chr<json_T>()) {
Expand All @@ -443,16 +443,18 @@ inline simdjson::simdjson_result<simdjson::dom::element> parse(simdjson::dom::pa
parser, /* ... and decompress to a RawVector if so, then parse that */
utils::decompress(std::string(json), Rcpp::String(std::string(*file_type))));
}
return parser.load(std::string(json)); /* otherwise, just `parser::load()` the file */
auto content = simdjson::padded_string::load(std::string_view(json));
return parser.iterate(content); /* otherwise, load the file and iterate it*/
} else {
return parser.parse(std::string_view(json)); /* if not file, just parse the string */
simdjson::padded_string content = simdjson::padded_string(std::string_view(json));
return parser.iterate(content); /* if not file, just parse the string */
}
}
}


template <bool query_error_ok>
inline SEXP query_and_deserialize(simdjson::dom::element parsed,
inline SEXP query_and_deserialize(simdjson::ondemand::value parsed,
const Rcpp::String::const_StringProxy& query,
SEXP on_query_error,
const rcppsimdjson::deserialize::Parse_Opts& parse_opts) {
Expand All @@ -466,14 +468,14 @@ inline SEXP query_and_deserialize(simdjson::dom::element p
}

if constexpr (query_error_ok) {
simdjson::dom::element queried;
simdjson::ondemand::value queried;
if(parsed.at_pointer(std::string_view(query)).get(queried) == simdjson::SUCCESS) {
return deserialize(queried, parse_opts); // #nocov
}
return on_query_error;

} else { /* !query_error_ok */
simdjson::dom::element queried;
simdjson::ondemand::value queried;
auto error = parsed.at_pointer(std::string_view(query)).get(queried);
if (error != simdjson::SUCCESS) {
Rcpp::stop(simdjson::error_message(error));
Expand All @@ -484,7 +486,7 @@ inline SEXP query_and_deserialize(simdjson::dom::element p


template <typename json_T, bool is_file, bool parse_error_ok>
inline SEXP parse_and_deserialize(simdjson::dom::parser& parser,
inline SEXP parse_and_deserialize(simdjson::ondemand::parser& parser,
const json_T& json,
SEXP on_parse_error,
const rcppsimdjson::deserialize::Parse_Opts& parse_opts) {
Expand All @@ -493,14 +495,14 @@ inline SEXP parse_and_deserialize(simdjson::dom::parser& p
}

if constexpr (parse_error_ok) {
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
if(simdjson::SUCCESS == parse<json_T, is_file>(parser, json).get(parsed)) {
return deserialize(parsed, parse_opts);
}
return on_parse_error;

} else {
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
auto error = parse<json_T, is_file>(parser, json).get(parsed);
if (error != simdjson::SUCCESS) {
Rcpp::stop(simdjson::error_message(error));
Expand All @@ -511,7 +513,7 @@ inline SEXP parse_and_deserialize(simdjson::dom::parser& p


template <typename json_T, bool is_file, bool parse_error_ok, bool query_error_ok>
inline SEXP parse_query_and_deserialize(simdjson::dom::parser& parser,
inline SEXP parse_query_and_deserialize(simdjson::ondemand::parser& parser,
const json_T& json,
const Rcpp::String::const_StringProxy& query,
SEXP on_parse_error,
Expand All @@ -522,14 +524,14 @@ inline SEXP parse_query_and_deserialize(simdjson::dom::parser& p
}

if constexpr (parse_error_ok) {
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
if(simdjson::SUCCESS == parse<json_T, is_file>(parser, json).get(parsed)) {
return query_and_deserialize<query_error_ok>(parsed, query, on_query_error, parse_opts);
}
return on_parse_error;

} else {
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
auto error = parse<json_T, is_file>(parser, json).get(parsed);
if (error != simdjson::SUCCESS) {
Rcpp::stop(simdjson::error_message(error));
Expand All @@ -547,7 +549,7 @@ template <typename json_T,
inline SEXP no_query(const json_T& json,
SEXP on_parse_error,
const rcppsimdjson::deserialize::Parse_Opts& parse_opts) {
simdjson::dom::parser parser;
simdjson::ondemand::parser parser;

if constexpr (is_single_json) {
return parse_and_deserialize<json_T, is_file, parse_error_ok>(
Expand Down Expand Up @@ -579,7 +581,7 @@ inline SEXP flat_query(const json_T& json,
SEXP on_parse_error,
SEXP on_query_error,
const rcppsimdjson::deserialize::Parse_Opts& parse_opts) {
simdjson::dom::parser parser;
simdjson::ondemand::parser parser;

if constexpr (is_single_json) {
if constexpr (is_single_query) {
Expand All @@ -591,7 +593,7 @@ inline SEXP flat_query(const json_T& json,
Rcpp::List out(n);

if constexpr (parse_error_ok) {
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
if(simdjson::SUCCESS == parse<json_T, is_file>(parser, json).get(parsed)) {
for (R_xlen_t i = 0; i < n; ++i) { // #nocov start
out[i] = query_and_deserialize<query_error_ok>(
Expand All @@ -603,7 +605,7 @@ inline SEXP flat_query(const json_T& json,
return on_parse_error;

} else { /* !parse_error_ok */
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
auto error = parse<json_T, is_file>(parser, json).get(parsed);
if (error != simdjson::SUCCESS) {
Rcpp::stop(simdjson::error_message(error));
Expand Down Expand Up @@ -668,11 +670,11 @@ inline SEXP nested_query(const json_T& json,
const rcppsimdjson::deserialize::Parse_Opts& parse_opts) {
const R_xlen_t n = std::size(json); /* query already checked to be the same size */
Rcpp::List out(n);
simdjson::dom::parser parser;
simdjson::ondemand::parser parser;

if constexpr (is_single_json) {
if constexpr (parse_error_ok) {
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
if(simdjson::SUCCESS == parse<json_T, is_file>(parser, json).get(parsed)) {
for (R_xlen_t i = 0; i < n; ++i) {
const R_xlen_t n_queries = std::size(query[i]);
Expand All @@ -689,7 +691,7 @@ inline SEXP nested_query(const json_T& json,
return on_parse_error;

} else { /* !parse_error_ok */
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
auto error = parse<json_T, is_file>(parser, json).get(parsed); // #nocov
if (error != simdjson::SUCCESS) {
Rcpp::stop(simdjson::error_message(error)); // #nocov
Expand All @@ -710,7 +712,7 @@ inline SEXP nested_query(const json_T& json,
for (R_xlen_t i = 0; i < n; ++i) {
const R_xlen_t n_queries = std::size(query[i]);
if constexpr (parse_error_ok) {
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
if(simdjson::SUCCESS == parse<decltype(json[i]), is_file>(parser, json[i]).get(parsed)) {
Rcpp::List res(n_queries); // #nocov start
for (R_xlen_t j = 0; j < n_queries; ++j) {
Expand All @@ -723,7 +725,7 @@ inline SEXP nested_query(const json_T& json,
out[i] = on_parse_error;

} else { /* !parse_error_ok */
simdjson::dom::element parsed;
simdjson::ondemand::value parsed;
auto error = parse<decltype(json[i]), is_file>(parser, json[i]).get(parsed);
if (error != simdjson::SUCCESS) {
Rcpp::stop(simdjson::error_message(error));
Expand Down
Loading