From e5e476d177ff892e0601fe04c906d83d60f93dec Mon Sep 17 00:00:00 2001 From: Liubov Didkivska Date: Wed, 10 Jun 2020 11:59:59 +0300 Subject: [PATCH 1/3] Add caching versions infos Add implementation to put/get versions list to catalog client cache. Implement versions infos cache tests. Relates-To: OLPEDGE-1606 Signed-off-by: Liubov Didkivska --- .../serializer/VersionInfosSerializer.cpp | 53 ++++++++++++++++ .../serializer/VersionInfosSerializer.h | 40 ++++++++++++ .../repositories/CatalogCacheRepository.cpp | 39 +++++++++++- .../src/repositories/CatalogCacheRepository.h | 9 ++- .../tests/CatalogCacheRepositoryTest.cpp | 61 +++++++++++++++++-- 5 files changed, 194 insertions(+), 8 deletions(-) create mode 100644 olp-cpp-sdk-dataservice-read/src/generated/serializer/VersionInfosSerializer.cpp create mode 100644 olp-cpp-sdk-dataservice-read/src/generated/serializer/VersionInfosSerializer.h diff --git a/olp-cpp-sdk-dataservice-read/src/generated/serializer/VersionInfosSerializer.cpp b/olp-cpp-sdk-dataservice-read/src/generated/serializer/VersionInfosSerializer.cpp new file mode 100644 index 000000000..d24b124da --- /dev/null +++ b/olp-cpp-sdk-dataservice-read/src/generated/serializer/VersionInfosSerializer.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2020 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +#include "VersionInfosSerializer.h" + +#include + +namespace olp { +namespace serializer { + +void to_json(const dataservice::read::model::VersionDependency& x, + rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator) { + value.SetObject(); + serialize("hrn", x.GetHrn(), value, allocator); + serialize("version", x.GetVersion(), value, allocator); + serialize("direct", x.GetDirect(), value, allocator); +} + +void to_json(const dataservice::read::model::VersionInfo& x, + rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator) { + value.SetObject(); + serialize("dependencies", x.GetDependencies(), value, allocator); + serialize("timestamp", x.GetTimestamp(), value, allocator); + serialize("version", x.GetVersion(), value, allocator); + serialize("partitionCounts", x.GetPartitionCounts(), value, allocator); +} + +void to_json(const dataservice::read::model::VersionInfos& x, + rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator) { + value.SetObject(); + serialize("versions", x.GetVersions(), value, allocator); +} +} // namespace serializer +} // namespace olp diff --git a/olp-cpp-sdk-dataservice-read/src/generated/serializer/VersionInfosSerializer.h b/olp-cpp-sdk-dataservice-read/src/generated/serializer/VersionInfosSerializer.h new file mode 100644 index 000000000..2d950baf0 --- /dev/null +++ b/olp-cpp-sdk-dataservice-read/src/generated/serializer/VersionInfosSerializer.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2020 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +#pragma once + +#include +#include "olp/dataservice/read/model/VersionInfos.h" + +namespace olp { +namespace serializer { + +void to_json(const dataservice::read::model::VersionDependency& x, + rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator); + +void to_json(const dataservice::read::model::VersionInfo& x, + rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator); + +void to_json(const dataservice::read::model::VersionInfos& x, + rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator); +} // namespace serializer +} // namespace olp diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp index c5ad00725..4ebc482ed 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 HERE Europe B.V. + * Copyright (C) 2019-2020 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,17 +19,21 @@ #include "CatalogCacheRepository.h" +#include #include +#include #include #include // clang-format off #include "generated/parser/CatalogParser.h" +#include "generated/parser/VersionInfosParser.h" #include "generated/parser/VersionResponseParser.h" #include #include "generated/serializer/CatalogSerializer.h" #include "generated/serializer/VersionResponseSerializer.h" +#include "generated/serializer/VersionInfosSerializer.h" #include "generated/serializer/JsonSerializer.h" // clang-format on @@ -46,6 +50,11 @@ std::string CreateKey(const std::string& hrn) { return hrn + "::catalog"; } std::string VersionKey(const std::string& hrn) { return hrn + "::latestVersion"; } +std::string VersionInfosKey(const std::string& hrn, std::int64_t start, + std::int64_t end) { + return hrn + "::" + std::to_string(start) + "::" + std::to_string(end) + + "::versionInfos"; +} time_t ConvertTime(std::chrono::seconds time) { return time == kChronoSecondsMax ? kTimetMax : time.count(); @@ -111,6 +120,34 @@ boost::optional CatalogCacheRepository::GetVersion() { return boost::any_cast(cached_version); } +void CatalogCacheRepository::PutVersionInfos( + std::int64_t start, std::int64_t end, const model::VersionInfos& versions) { + std::string hrn(hrn_.ToCatalogHRNString()); + auto key = VersionInfosKey(hrn, start, end); + OLP_SDK_LOG_DEBUG_F(kLogTag, "PutVersionInfos -> '%s'", key.c_str()); + auto list_versions = olp::serializer::serialize(versions); + + auto versions_data = std::make_shared>( + std::begin(list_versions), std::end(list_versions)); + + cache_->Put(VersionInfosKey(hrn, start, end), versions_data, default_expiry_); +} + +boost::optional CatalogCacheRepository::GetVersionInfos( + std::int64_t start, std::int64_t end) { + std::string hrn(hrn_.ToCatalogHRNString()); + auto key = VersionInfosKey(hrn, start, end); + OLP_SDK_LOG_DEBUG_F(kLogTag, "GetVersionInfos -> '%s'", key.c_str()); + + auto value = cache_->Get(key); + if (!value) { + return boost::none; + } + + auto list_versions = std::string(value->begin(), value->end()); + return parser::parse(list_versions); +} + void CatalogCacheRepository::Clear() { std::string hrn(hrn_.ToCatalogHRNString()); OLP_SDK_LOG_INFO_F(kLogTag, "Clear -> '%s'", CreateKey(hrn).c_str()); diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.h b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.h index 6352ba5de..b5128cd2e 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.h +++ b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 HERE Europe B.V. + * Copyright (C) 2019-2020 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -51,6 +52,12 @@ class CatalogCacheRepository final { boost::optional GetVersion(); + void PutVersionInfos(std::int64_t start, std::int64_t end, + const model::VersionInfos& versions); + + boost::optional GetVersionInfos(std::int64_t start, + std::int64_t end); + void Clear(); private: diff --git a/olp-cpp-sdk-dataservice-read/tests/CatalogCacheRepositoryTest.cpp b/olp-cpp-sdk-dataservice-read/tests/CatalogCacheRepositoryTest.cpp index 1788fd22d..62dc8d317 100644 --- a/olp-cpp-sdk-dataservice-read/tests/CatalogCacheRepositoryTest.cpp +++ b/olp-cpp-sdk-dataservice-read/tests/CatalogCacheRepositoryTest.cpp @@ -26,15 +26,16 @@ namespace { -using namespace olp; -using namespace olp::dataservice::read; +namespace read = olp::dataservice::read; +namespace client = olp::client; +namespace cache = olp::cache; constexpr auto kCatalog = "hrn:here:data::olp-here-test:catalog"; -TEST(PartitionsCacheRepositoryTest, DefaultExpiry) { +TEST(CatalogCacheRepositoryTest, DefaultExpiry) { const auto hrn = client::HRN::FromString(kCatalog); - model::Catalog model_catalog; + read::model::Catalog model_catalog; { SCOPED_TRACE("Disable expiration"); @@ -42,7 +43,8 @@ TEST(PartitionsCacheRepositoryTest, DefaultExpiry) { const auto default_expiry = std::chrono::seconds::max(); std::shared_ptr cache = olp::client::OlpClientSettingsFactory::CreateDefaultCache({}); - repository::CatalogCacheRepository repository(hrn, cache, default_expiry); + read::repository::CatalogCacheRepository repository(hrn, cache, + default_expiry); repository.Put(model_catalog); const auto result = repository.Get(); @@ -56,7 +58,8 @@ TEST(PartitionsCacheRepositoryTest, DefaultExpiry) { const auto default_expiry = std::chrono::seconds(-1); std::shared_ptr cache = olp::client::OlpClientSettingsFactory::CreateDefaultCache({}); - repository::CatalogCacheRepository repository(hrn, cache, default_expiry); + read::repository::CatalogCacheRepository repository(hrn, cache, + default_expiry); repository.Put(model_catalog); const auto result = repository.Get(); @@ -65,4 +68,50 @@ TEST(PartitionsCacheRepositoryTest, DefaultExpiry) { } } +TEST(CatalogCacheRepositoryTest, VersionsList) { + const auto hrn = client::HRN::FromString(kCatalog); + + read::model::VersionInfos model_versions; + model_versions.SetVersions( + std::vector(1)); + const auto default_expiry = std::chrono::seconds::max(); + std::shared_ptr cache = + olp::client::OlpClientSettingsFactory::CreateDefaultCache({}); + read::repository::CatalogCacheRepository repository(hrn, cache, + default_expiry); + + { + SCOPED_TRACE("Put/get versions list"); + + repository.PutVersionInfos(3, 4, model_versions); + const auto result = repository.GetVersionInfos(3, 4); + + EXPECT_TRUE(result); + EXPECT_EQ(1u, result->GetVersions().size()); + } + + { + SCOPED_TRACE("Get versions list wrong key"); + + const auto result = repository.GetVersionInfos(300, 3001); + + EXPECT_FALSE(result); + } + + { + SCOPED_TRACE("List versions expired"); + + const auto default_expiry = std::chrono::seconds(-1); + std::shared_ptr cache_expiration = + olp::client::OlpClientSettingsFactory::CreateDefaultCache({}); + read::repository::CatalogCacheRepository repository_expiration( + hrn, cache_expiration, default_expiry); + + repository_expiration.PutVersionInfos(3, 4, model_versions); + const auto result = repository_expiration.GetVersionInfos(3, 4); + + EXPECT_FALSE(result); + } +} + } // namespace From 5a9c5eedaeccd98c9d929a7199f18dcf12a688d8 Mon Sep 17 00:00:00 2001 From: Liubov Didkivska Date: Wed, 10 Jun 2020 14:10:27 +0300 Subject: [PATCH 2/3] Add caching versions infos Add implementation to put/get versions list to catalog client cache. Implement versions infos cache tests. Relates-To: OLPEDGE-1606 Signed-off-by: Liubov Didkivska --- .../generated/parser/JsonParserCacheValue.h | 41 +++++++++++++++++++ .../repositories/CatalogCacheRepository.cpp | 10 ++--- 2 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 olp-cpp-sdk-dataservice-read/src/generated/parser/JsonParserCacheValue.h diff --git a/olp-cpp-sdk-dataservice-read/src/generated/parser/JsonParserCacheValue.h b/olp-cpp-sdk-dataservice-read/src/generated/parser/JsonParserCacheValue.h new file mode 100644 index 000000000..f2d0552cd --- /dev/null +++ b/olp-cpp-sdk-dataservice-read/src/generated/parser/JsonParserCacheValue.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2020 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +#pragma once + +#include +#include + +namespace olp { +namespace parser { +template +inline T parse(const cache::KeyValueCache::ValueType& cached_json) { + rapidjson::Document doc; + T result{}; + if (!cached_json.empty() && cached_json.back() == '\0') { + doc.Parse(reinterpret_cast(cached_json.data())); + if (doc.IsObject() || doc.IsArray()) { + from_json(doc, result); + } + } + return result; +} + +} // namespace parser +} // namespace olp diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp index 4ebc482ed..eb67fc5f5 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp @@ -30,6 +30,7 @@ #include "generated/parser/CatalogParser.h" #include "generated/parser/VersionInfosParser.h" #include "generated/parser/VersionResponseParser.h" +#include "generated/parser/JsonParserCacheValue.h" #include #include "generated/serializer/CatalogSerializer.h" #include "generated/serializer/VersionResponseSerializer.h" @@ -127,9 +128,9 @@ void CatalogCacheRepository::PutVersionInfos( OLP_SDK_LOG_DEBUG_F(kLogTag, "PutVersionInfos -> '%s'", key.c_str()); auto list_versions = olp::serializer::serialize(versions); - auto versions_data = std::make_shared>( - std::begin(list_versions), std::end(list_versions)); - + // include null terminated symbol + auto versions_data = std::make_shared( + list_versions.data(), list_versions.data() + list_versions.size() + 1); cache_->Put(VersionInfosKey(hrn, start, end), versions_data, default_expiry_); } @@ -144,8 +145,7 @@ boost::optional CatalogCacheRepository::GetVersionInfos( return boost::none; } - auto list_versions = std::string(value->begin(), value->end()); - return parser::parse(list_versions); + return parser::parse(*value); } void CatalogCacheRepository::Clear() { From 0ef9d587d4ea24d21b3ffda6224ce95402650094 Mon Sep 17 00:00:00 2001 From: Liubov Didkivska Date: Wed, 10 Jun 2020 17:23:43 +0300 Subject: [PATCH 3/3] Add caching versions infos Add implementation to put/get versions list to catalog client cache. Implement versions infos cache tests. Relates-To: OLPEDGE-1606 Signed-off-by: Liubov Didkivska --- .../src/generated/parser/JsonParserCacheValue.h | 5 +++-- .../src/repositories/CatalogCacheRepository.cpp | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/olp-cpp-sdk-dataservice-read/src/generated/parser/JsonParserCacheValue.h b/olp-cpp-sdk-dataservice-read/src/generated/parser/JsonParserCacheValue.h index f2d0552cd..78cae2c87 100644 --- a/olp-cpp-sdk-dataservice-read/src/generated/parser/JsonParserCacheValue.h +++ b/olp-cpp-sdk-dataservice-read/src/generated/parser/JsonParserCacheValue.h @@ -28,8 +28,9 @@ template inline T parse(const cache::KeyValueCache::ValueType& cached_json) { rapidjson::Document doc; T result{}; - if (!cached_json.empty() && cached_json.back() == '\0') { - doc.Parse(reinterpret_cast(cached_json.data())); + if (!cached_json.empty()) { + auto* data = static_cast(cached_json.data()); + doc.Parse(static_cast(data), cached_json.size()); if (doc.IsObject() || doc.IsArray()) { from_json(doc, result); } diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp index eb67fc5f5..bce018056 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp @@ -128,9 +128,8 @@ void CatalogCacheRepository::PutVersionInfos( OLP_SDK_LOG_DEBUG_F(kLogTag, "PutVersionInfos -> '%s'", key.c_str()); auto list_versions = olp::serializer::serialize(versions); - // include null terminated symbol auto versions_data = std::make_shared( - list_versions.data(), list_versions.data() + list_versions.size() + 1); + list_versions.data(), list_versions.data() + list_versions.size()); cache_->Put(VersionInfosKey(hrn, start, end), versions_data, default_expiry_); }