diff --git a/cpp/cmake_modules/FindsimdjsonAlt.cmake b/cpp/cmake_modules/FindsimdjsonAlt.cmake new file mode 100644 index 000000000000..966f3d039455 --- /dev/null +++ b/cpp/cmake_modules/FindsimdjsonAlt.cmake @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(SimdjsonAlt_FOUND) + return() +endif() + +# Try to find simdjson manually first, as some system packages have broken +# CMake configs that reference non-existent library files (e.g., Alpine Linux) +if(simdjson_ROOT) + find_path(SIMDJSON_INCLUDE_DIR + NAMES simdjson.h + PATHS ${simdjson_ROOT} + NO_DEFAULT_PATH + PATH_SUFFIXES "include") + find_library(SIMDJSON_LIBRARY + NAMES simdjson + PATHS ${simdjson_ROOT} + NO_DEFAULT_PATH + PATH_SUFFIXES "lib" "lib64") +else() + find_path(SIMDJSON_INCLUDE_DIR + NAMES simdjson.h + PATH_SUFFIXES "include") + find_library(SIMDJSON_LIBRARY + NAMES simdjson + PATH_SUFFIXES "lib" "lib64") +endif() + +if(SIMDJSON_INCLUDE_DIR AND SIMDJSON_LIBRARY) + # Found via manual search + file(READ "${SIMDJSON_INCLUDE_DIR}/simdjson.h" SIMDJSON_H_CONTENT) + string(REGEX MATCH "#define SIMDJSON_VERSION \"([0-9]+\\.[0-9]+\\.[0-9]+)\"" + SIMDJSON_VERSION_DEFINITION "${SIMDJSON_H_CONTENT}") + string(REGEX REPLACE "^.+ \"([0-9]+\\.[0-9]+\\.[0-9]+)\"$" "\\1" SIMDJSON_VERSION + "${SIMDJSON_VERSION_DEFINITION}") + if("${SIMDJSON_VERSION}" STREQUAL "") + set(SIMDJSON_VERSION "0.0.0") + endif() + + find_package_handle_standard_args( + SimdjsonAlt + REQUIRED_VARS SIMDJSON_INCLUDE_DIR SIMDJSON_LIBRARY + VERSION_VAR SIMDJSON_VERSION) + + if(SimdjsonAlt_FOUND) + if(WIN32 AND "${SIMDJSON_INCLUDE_DIR}" MATCHES "^/") + # MSYS2 + execute_process(COMMAND "cygpath" "--windows" "${SIMDJSON_INCLUDE_DIR}" + OUTPUT_VARIABLE SIMDJSON_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE) + endif() + # Detect library type based on file extension + if("${SIMDJSON_LIBRARY}" MATCHES "\\.(so|dylib)(\\.[0-9]+)*$" + OR "${SIMDJSON_LIBRARY}" MATCHES "\\.dll$") + add_library(simdjson::simdjson SHARED IMPORTED) + else() + add_library(simdjson::simdjson STATIC IMPORTED) + endif() + set_target_properties(simdjson::simdjson + PROPERTIES IMPORTED_LOCATION "${SIMDJSON_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES + "${SIMDJSON_INCLUDE_DIR}") + endif() + return() +endif() + +# Manual search failed, try CMake config mode +set(find_package_args) +if(SimdjsonAlt_FIND_VERSION) + list(APPEND find_package_args ${SimdjsonAlt_FIND_VERSION}) +endif() +if(SimdjsonAlt_FIND_QUIETLY) + list(APPEND find_package_args QUIET) +endif() +find_package(simdjson ${find_package_args} CONFIG) +if(simdjson_FOUND) + set(SimdjsonAlt_FOUND TRUE) + if(NOT TARGET simdjson::simdjson) + # simdjson's CMake config should create this target, but create it if missing + if(TARGET simdjson) + add_library(simdjson::simdjson ALIAS simdjson) + endif() + endif() +endif() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 07054b4bb295..68333fedaaaa 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -64,6 +64,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES re2 Protobuf RapidJSON + simdjson Snappy Substrait Thrift @@ -137,6 +138,10 @@ if(ARROW_DEPENDENCY_SOURCE STREQUAL "CONDA") if("${GTest_SOURCE}" STREQUAL "") set(GTest_SOURCE "AUTO") endif() + # simdjson is not commonly available in conda, so we allow auto fallback. + if("${simdjson_SOURCE}" STREQUAL "") + set(simdjson_SOURCE "AUTO") + endif() message(STATUS "Using CONDA_PREFIX for ARROW_PACKAGE_PREFIX: ${ARROW_PACKAGE_PREFIX}") else() set(ARROW_ACTUAL_DEPENDENCY_SOURCE "${ARROW_DEPENDENCY_SOURCE}") @@ -209,6 +214,8 @@ macro(build_dependency DEPENDENCY_NAME) build_protobuf() elseif("${DEPENDENCY_NAME}" STREQUAL "RapidJSON") build_rapidjson() + elseif("${DEPENDENCY_NAME}" STREQUAL "simdjson") + build_simdjson() elseif("${DEPENDENCY_NAME}" STREQUAL "re2") build_re2() elseif("${DEPENDENCY_NAME}" STREQUAL "Snappy") @@ -411,6 +418,10 @@ if(ARROW_JSON OR ARROW_FLIGHT_SQL_ODBC) set(ARROW_WITH_RAPIDJSON ON) endif() +if(ARROW_JSON) + set(ARROW_WITH_SIMDJSON ON) +endif() + if(ARROW_ORC OR ARROW_FLIGHT) set(ARROW_WITH_PROTOBUF ON) endif() @@ -763,6 +774,15 @@ else() "${THIRDPARTY_MIRROR_URL}/rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz") endif() +if(DEFINED ENV{ARROW_SIMDJSON_URL}) + set(SIMDJSON_SOURCE_URL "$ENV{ARROW_SIMDJSON_URL}") +else() + set_urls( + SIMDJSON_SOURCE_URL + "https://github.com/simdjson/simdjson/archive/refs/tags/${ARROW_SIMDJSON_BUILD_VERSION}.tar.gz" + "${THIRDPARTY_MIRROR_URL}/simdjson-${ARROW_SIMDJSON_BUILD_VERSION}.tar.gz") +endif() + if(DEFINED ENV{ARROW_S2N_TLS_URL}) set(S2N_TLS_SOURCE_URL "$ENV{ARROW_S2N_TLS_URL}") else() @@ -2782,6 +2802,61 @@ if(ARROW_BUILD_BENCHMARKS) FALSE) endif() +macro(build_simdjson) + message(STATUS "Building simdjson from source") + set(SIMDJSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/simdjson_ep/src/simdjson_ep-install") + set(SIMDJSON_INCLUDE_DIR "${SIMDJSON_PREFIX}/include") + set(SIMDJSON_LIB_DIR "${SIMDJSON_PREFIX}/lib") + + set(SIMDJSON_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + -DSIMDJSON_BUILD_STATIC_LIB=ON + -DSIMDJSON_DEVELOPER_MODE=OFF + -DSIMDJSON_ENABLE_THREADS=ON + -DBUILD_SHARED_LIBS=OFF + "-DCMAKE_INSTALL_PREFIX=${SIMDJSON_PREFIX}") + + set(SIMDJSON_STATIC_LIB + "${SIMDJSON_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}simdjson${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + externalproject_add(simdjson_ep + ${EP_COMMON_OPTIONS} + PREFIX "${CMAKE_BINARY_DIR}" + URL ${SIMDJSON_SOURCE_URL} + URL_HASH "SHA256=${ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${SIMDJSON_CMAKE_ARGS} + BUILD_BYPRODUCTS "${SIMDJSON_STATIC_LIB}") + + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${SIMDJSON_INCLUDE_DIR}") + + # Check if target already exists (may have been created by find_package with incompatible version) + if(NOT TARGET simdjson::simdjson) + add_library(simdjson::simdjson STATIC IMPORTED) + endif() + set_target_properties(simdjson::simdjson + PROPERTIES IMPORTED_LOCATION "${SIMDJSON_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${SIMDJSON_INCLUDE_DIR}") + add_dependencies(simdjson::simdjson simdjson_ep) + + set(SIMDJSON_VENDORED TRUE) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS simdjson::simdjson) +endmacro() + +if(ARROW_WITH_SIMDJSON) + set(ARROW_SIMDJSON_REQUIRED_VERSION "3.0.0") + resolve_dependency(simdjson + HAVE_ALT + TRUE + REQUIRED_VERSION + ${ARROW_SIMDJSON_REQUIRED_VERSION} + IS_RUNTIME_DEPENDENCY + FALSE) +endif() + function(build_rapidjson) list(APPEND CMAKE_MESSAGE_INDENT "RapidJSON: ") message(STATUS "Building from source") diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 8750598f6c3b..e792574a1837 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -1044,7 +1044,7 @@ if(ARROW_JSON) json/parser.cc json/reader.cc) foreach(ARROW_JSON_TARGET ${ARROW_JSON_TARGETS}) - target_link_libraries(${ARROW_JSON_TARGET} PRIVATE RapidJSON) + target_link_libraries(${ARROW_JSON_TARGET} PRIVATE RapidJSON simdjson::simdjson) endforeach() else() set(ARROW_JSON_TARGET_SHARED) diff --git a/cpp/src/arrow/json/CMakeLists.txt b/cpp/src/arrow/json/CMakeLists.txt index fa7d06078486..b5b47c0ed37b 100644 --- a/cpp/src/arrow/json/CMakeLists.txt +++ b/cpp/src/arrow/json/CMakeLists.txt @@ -23,6 +23,7 @@ add_arrow_test(test from_string_test.cc parser_test.cc reader_test.cc + object_parser_test.cc PREFIX "arrow-json" EXTRA_LINK_LIBS diff --git a/cpp/src/arrow/json/object_parser.cc b/cpp/src/arrow/json/object_parser.cc index 346fd1039a2b..eee151fb5569 100644 --- a/cpp/src/arrow/json/object_parser.cc +++ b/cpp/src/arrow/json/object_parser.cc @@ -16,72 +16,116 @@ // under the License. #include "arrow/json/object_parser.h" -#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep -#include +#include namespace arrow { namespace json { namespace internal { -namespace rj = arrow::rapidjson; - class ObjectParser::Impl { public: Status Parse(std::string_view json) { - document_.Parse(reinterpret_cast(json.data()), - static_cast(json.size())); + // Copy into padded buffer + padded_json_ = simdjson::padded_string(json); + + // Parse + auto result = parser_.parse(padded_json_); - if (document_.HasParseError()) { - return Status::Invalid("Json parse error (offset ", document_.GetErrorOffset(), - "): ", document_.GetParseError()); + // Handle parse errors + if (result.error()) { + return Status::Invalid("Json parse error: ", + simdjson::error_message(result.error())); } - if (!document_.IsObject()) { + + // Store parsed document + document_ = std::move(result).value(); + + // Validate root is an object + if (!document_.is_object()) { return Status::TypeError("Not a json object"); } + return Status::OK(); } Result GetString(const char* key) const { - if (!document_.HasMember(key)) { + auto field = document_[key]; + if (field.error() == simdjson::NO_SUCH_FIELD) { return Status::KeyError("Key '", key, "' does not exist"); } - if (!document_[key].IsString()) { + if (field.error()) { + return Status::Invalid("Error accessing key '", key, + "': ", simdjson::error_message(field.error())); + } + + auto str_result = field.get_string(); + if (str_result.error() == simdjson::INCORRECT_TYPE) { return Status::TypeError("Key '", key, "' is not a string"); } - return document_[key].GetString(); + if (str_result.error()) { + return Status::Invalid("Error getting string for key '", key, + "': ", simdjson::error_message(str_result.error())); + } + + return std::string(str_result.value()); } Result> GetStringMap() const { std::unordered_map map; - for (auto itr = document_.MemberBegin(); itr != document_.MemberEnd(); ++itr) { - const auto& json_name = itr->name; - const auto& json_value = itr->value; - if (!json_name.IsString()) { - return Status::TypeError("Key is not a string"); + + auto obj_result = document_.get_object(); + if (obj_result.error()) { + return Status::TypeError("Document is not an object"); + } + + simdjson::dom::object obj = obj_result.value(); + + for (auto [key, value] : obj) { + auto str_result = value.get_string(); + + if (str_result.error() == simdjson::INCORRECT_TYPE) { + return Status::TypeError("Key '", std::string(key), + "' does not have a string value"); } - std::string name = json_name.GetString(); - if (!json_value.IsString()) { - return Status::TypeError("Key '", name, "' does not have a string value"); + if (str_result.error()) { + return Status::Invalid("Error getting value for key '", + std::string(key), "': ", + simdjson::error_message(str_result.error())); } - std::string value = json_value.GetString(); - map.insert({std::move(name), std::move(value)}); + + map.emplace(std::string(key), std::string(str_result.value())); } + return map; } Result GetBool(const char* key) const { - if (!document_.HasMember(key)) { + auto field = document_[key]; + if (field.error() == simdjson::NO_SUCH_FIELD) { return Status::KeyError("Key '", key, "' does not exist"); } - if (!document_[key].IsBool()) { + if (field.error()) { + return Status::Invalid("Error accessing key '", key, + "': ", simdjson::error_message(field.error())); + } + + auto bool_result = field.get_bool(); + if (bool_result.error() == simdjson::INCORRECT_TYPE) { return Status::TypeError("Key '", key, "' is not a boolean"); } - return document_[key].GetBool(); + if (bool_result.error()) { + return Status::Invalid("Error getting bool for key '", key, + "': ", simdjson::error_message(bool_result.error())); + } + + return bool_result.value(); } private: - rj::Document document_; + simdjson::dom::parser parser_; + simdjson::padded_string padded_json_; + simdjson::dom::element document_; }; ObjectParser::ObjectParser() : impl_(new ObjectParser::Impl()) {} diff --git a/cpp/src/arrow/json/object_parser_test.cc b/cpp/src/arrow/json/object_parser_test.cc new file mode 100644 index 000000000000..6de7d550b3c6 --- /dev/null +++ b/cpp/src/arrow/json/object_parser_test.cc @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/json/object_parser.h" + +#include +#include + +#include +#include + +#include "arrow/testing/gtest_util.h" + +namespace arrow { +namespace json { +namespace internal { + +TEST(ObjectParser, GetString) { + ObjectParser parser; + + ASSERT_OK(parser.Parse(R"({"name":"arrow"})")); + + ASSERT_OK_AND_ASSIGN(auto value, parser.GetString("name")); + EXPECT_EQ(value, "arrow"); +} + +TEST(ObjectParser, GetBool) { + ObjectParser parser; + + ASSERT_OK(parser.Parse(R"({"enabled":true})")); + + ASSERT_OK_AND_ASSIGN(auto value, parser.GetBool("enabled")); + EXPECT_TRUE(value); +} + +TEST(ObjectParser, InvalidJson) { + ObjectParser parser; + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr("Json parse error"), + parser.Parse(R"({"name":)")); +} + +TEST(ObjectParser, GetStringMap) { + ObjectParser parser; + + ASSERT_OK(parser.Parse(R"({ + "k1": "v1", + "k2": "v2" + })")); + + ASSERT_OK_AND_ASSIGN(auto map, parser.GetStringMap()); + + ASSERT_EQ(map.size(), 2U); + EXPECT_EQ(map["k1"], "v1"); + EXPECT_EQ(map["k2"], "v2"); +} + +TEST(ObjectParser, MissingKey) { + ObjectParser parser; + + ASSERT_OK(parser.Parse(R"({ + "name": "arrow" + })")); + + ASSERT_RAISES(KeyError, parser.GetString("missing")); + ASSERT_RAISES(KeyError, parser.GetBool("missing")); +} + +TEST(ObjectParser, WrongType) { + ObjectParser parser; + + ASSERT_OK(parser.Parse(R"({ + "flag": true, + "name": "arrow" + })")); + + ASSERT_RAISES(TypeError, parser.GetString("flag")); + ASSERT_RAISES(TypeError, parser.GetBool("name")); +} + +TEST(ObjectParser, NonObjectRoot) { + ObjectParser parser; + + ASSERT_RAISES(TypeError, parser.Parse(R"(["a", "b"])")); +} + +TEST(ObjectParser, EmptyObject) { + ObjectParser parser; + + ASSERT_OK(parser.Parse(R"({})")); + + ASSERT_OK_AND_ASSIGN(auto map, parser.GetStringMap()); + + EXPECT_TRUE(map.empty()); +} + +} // namespace internal +} // namespace json +} // namespace arrow diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index c6f4b01a717c..f77440a826e5 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -102,6 +102,8 @@ ARROW_RAPIDJSON_BUILD_SHA256_CHECKSUM=b9290a9a6d444c8e049bd589ab804e0ccf2b05dc59 # for CRAN builds. This version includes musl libc support (GH-48010). ARROW_RE2_BUILD_VERSION=2023-03-01 ARROW_RE2_BUILD_SHA256_CHECKSUM=7a9a4824958586980926a300b4717202485c4b4115ac031822e29aa4ef207e48 +ARROW_SIMDJSON_BUILD_VERSION=v3.12.3 +ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM=d0af071f2f4187d8b26b556e83ef832b634bd5feb4e2f537b9dabbd334d4e334 ARROW_SNAPPY_BUILD_VERSION=1.2.2 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0 @@ -163,6 +165,7 @@ DEPENDENCIES=( "ARROW_ORC_URL orc-${ARROW_ORC_BUILD_VERSION}.tar.gz https://www.apache.org/dyn/closer.lua/orc/orc-${ARROW_ORC_BUILD_VERSION}/orc-${ARROW_ORC_BUILD_VERSION}.tar.gz?action=download" "ARROW_PROTOBUF_URL protobuf-${ARROW_PROTOBUF_BUILD_VERSION}.tar.gz https://github.com/google/protobuf/releases/download/${ARROW_PROTOBUF_BUILD_VERSION}/protobuf-${ARROW_PROTOBUF_BUILD_VERSION:1}.tar.gz" "ARROW_RAPIDJSON_URL rapidjson-${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz https://github.com/miloyip/rapidjson/archive/${ARROW_RAPIDJSON_BUILD_VERSION}.tar.gz" + "ARROW_SIMDJSON_URL simdjson-${ARROW_SIMDJSON_BUILD_VERSION}.tar.gz https://github.com/simdjson/simdjson/archive/refs/tags/${ARROW_SIMDJSON_BUILD_VERSION}.tar.gz" "ARROW_RE2_URL re2-${ARROW_RE2_BUILD_VERSION}.tar.gz https://github.com/google/re2/archive/${ARROW_RE2_BUILD_VERSION}.tar.gz" "ARROW_S2N_TLS_URL s2n-${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz https://github.com/aws/s2n-tls/archive/${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz" "ARROW_SNAPPY_URL snappy-${ARROW_SNAPPY_BUILD_VERSION}.tar.gz https://github.com/google/snappy/archive/${ARROW_SNAPPY_BUILD_VERSION}.tar.gz"