Add leveldb storage plugin support

leveldb_vendor/CMakeLists.txt

@@ -0,0 +1,42 @@
+cmake_minimum_required(VERSION 3.7)
+
+project(leveldb_vendor)
+
+find_package(ament_cmake REQUIRED)
+
+list(INSERT CMAKE_MODULE_PATH 0 "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules")
+find_package(leveldb QUIET)
+
+if (NOT leveldb_FOUND)
+  set(extra_cmake_args)
+
+  if(DEFINED CMAKE_TOOLCHAIN_FILE)
+    list(APPEND extra_cmake_args "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}")
+  endif()
+
+  find_package(Patch REQUIRED)
+  include(ExternalProject)
+  ExternalProject_Add(leveldb-1.22
+    PREFIX leveldb-1.22
+    URL https://github.com/google/leveldb/archive/1.22.zip
+    URL_MD5 f741bc416308adb35d79900afe282d9e
+    TIMEOUT 600
+    CMAKE_ARGS
+      -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/leveldb_install
+      -DBUILD_SHARED_LIBS:BOOL=ON
+      -DLEVELDB_BUILD_TESTS:BOOL=OFF
+      -DLEVELDB_BUILD_BENCHMARKS:BOOL=OFF
+      ${extra_cmake_args}
+    PATCH_COMMAND
+      ${Patch_EXECUTABLE} -p1 -N < ${CMAKE_CURRENT_SOURCE_DIR}/leveldb_cmakefile.diff
+    )
+  install(
+    DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/leveldb_install/
+    DESTINATION ${CMAKE_INSTALL_PREFIX})
+else()
+  message(STATUS "Found leveldb. Using leveldb from system.")
+endif()
+
+install(DIRECTORY cmake DESTINATION share/${PROJECT_NAME})
+
+ament_package(CONFIG_EXTRAS leveldb_vendor-extras.cmake)

leveldb_vendor/README.md

@@ -0,0 +1,3 @@
+# leveldb_vendor
+
+CMake wrapper for the [leveldb](https://github.com/google/leveldb) library.

leveldb_vendor/cmake/Modules/Findleveldb.cmake

@@ -0,0 +1,80 @@
+##################################################################################################
+#
+# CMake script for finding leveldb.
+#
+# Cache variables (not intended to be used in CMakeLists.txt files)
+#
+# - leveldb_INCLUDE_DIR: Absolute path to package headers.
+# - leveldb_LIBRARY: Absolute path to library.
+#
+#
+# Output variables:
+#
+# - leveldb_FOUND: Boolean that indicates if the package was found
+# - leveldb_INCLUDE_DIRS: Paths to the necessary header files
+# - leveldb_LIBRARIES: Package libraries
+#
+# Example usage:
+#
+#  find_package(leveldb)
+#  if(NOT leveldb_FOUND)
+#    # Error handling
+#  endif()
+#  ...
+#  include_directories(${leveldb_INCLUDE_DIRS} ...)
+#  ...
+#  target_link_libraries(my_target ${leveldb_LIBRARIES})
+#
+##################################################################################################
+
+find_package(Threads REQUIRED)
+find_package(leveldb CONFIG QUIET)
+
+if (NOT leveldb_FOUND)
+
+  # Get package location hint from environment variable (if any)
+  if(NOT leveldb_ROOT_DIR AND DEFINED ENV{leveldb_ROOT_DIR})
+      set(leveldb_ROOT_DIR "$ENV{leveldb_ROOT_DIR}" CACHE PATH
+          "leveldb base directory location (optional, used for nonstandard installation paths)")
+  endif()
+
+   # Search path for nonstandard package locations
+   if(leveldb_ROOT_DIR)
+       set(leveldb_INCLUDE_PATH PATHS "${leveldb_ROOT_DIR}/include" NO_DEFAULT_PATH)
+       set(leveldb_LIBRARY_PATH PATHS "${leveldb_ROOT_DIR}/lib"     NO_DEFAULT_PATH)
+   else()
+       set(leveldb_INCLUDE_PATH "")
+       set(leveldb_LIBRARY_PATH "")
+   endif()
+
+    # Find headers and libraries
+    find_path(leveldb_INCLUDE_DIR NAMES db.h PATH_SUFFIXES "leveldb" ${leveldb_INCLUDE_PATH})
+    find_library(leveldb_LIBRARY  NAMES leveldb PATH_SUFFIXES "leveldb" ${leveldb_LIBRARY_PATH})
+
+    mark_as_advanced(leveldb_INCLUDE_DIR leveldb_LIBRARY)
+
+    # Output variables generation
+    include(FindPackageHandleStandardArgs)
+    find_package_handle_standard_args(leveldb DEFAULT_MSG leveldb_LIBRARY leveldb_INCLUDE_DIR)
+
+    set(leveldb_FOUND ${LEVELDB_FOUND}) # Enforce case-correctness: Set appropriately cased variable...
+    unset(LEVELDB_FOUND) # ...and unset uppercase variable generated by find_package_handle_standard_args
+
+    if(leveldb_FOUND)
+        set(leveldb_INCLUDE_DIR ${leveldb_INCLUDE_DIR})
+        set(leveldb_LIBRARY ${leveldb_LIBRARY})
+
+        if(NOT TARGET leveldb::leveldb)
+            add_library(leveldb::leveldb UNKNOWN IMPORTED)
+            set_property(TARGET leveldb::leveldb PROPERTY IMPORTED_LOCATION ${leveldb_LIBRARY})
+            set_property(TARGET leveldb::leveldb PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${leveldb_INCLUDE_DIR})
+        endif()
+        list(APPEND leveldb_TARGETS leveldb::leveldb)
+    elseif(leveldb_FIND_REQUIRED)
+        message(FATAL_ERROR "Unable to find leveldb")
+    endif()
+
+else()
+    message(STATUS "Found leveldb from installed package in ${leveldb_DIR}")
+    list(APPEND leveldb_TARGETS leveldb::leveldb)
+endif()

leveldb_vendor/leveldb_cmakefile.diff

@@ -0,0 +1,13 @@
+diff -Nur a/CMakeLists.txt b/CMakeLists.txt
+--- a/CMakeLists.txt	2019-05-04 04:24:26.000000000 +0800
++++ b/CMakeLists.txt	2020-11-18 09:30:19.275999814 +0800
+@@ -94,6 +94,9 @@
+   add_compile_options(-fvisibility=hidden)
+ endif(BUILD_SHARED_LIBS)
+
++# Must be included before CMAKE_INSTALL_INCLUDEDIR is used.
++include(GNUInstallDirs)
++
+ add_library(leveldb "")
+ target_sources(leveldb
+   PRIVATE

leveldb_vendor/leveldb_vendor-extras.cmake

@@ -0,0 +1,15 @@
+# Copyright 2020 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+list(INSERT CMAKE_MODULE_PATH 0 "${leveldb_vendor_DIR}/Modules")

leveldb_vendor/package.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<?xml-model href="http://download.ros.org/schema/package_format2.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<package format="2">
+  <name>leveldb_vendor</name>
+  <version>0.0.1</version>
+  <description>leveldb vendor package, providing a dependency for leveldb.</description>
+  <maintainer email="barry.xu@sony.com">Barry Xu</maintainer>
+  <maintainer email="tomoya.fujita@sony.com">Tomoya Fujita</maintainer>
+  <license>Apache License 2.0</license>  <!-- the contents of this package are Apache 2.0 -->
+  <license>BSD</license>  <!-- leveldb is BSD -->
+
+  <url type="website">https://github.com/google/leveldb</url>
+
+  <buildtool_depend>ament_cmake</buildtool_depend>
+
+  <depend>snappy</depend>
+  <depend>leveldb</depend>
+
+  <export>
+    <build_type>ament_cmake</build_type>
+  </export>
+</package>

rosbag2_storage/src/rosbag2_storage/metadata_io.cpp

@@ -233,7 +233,7 @@ BagMetadata MetadataIo::read_metadata(const std::string & uri)
     auto metadata = yaml_file["rosbag2_bagfile_information"].as<rosbag2_storage::BagMetadata>();
     rcutils_allocator_t allocator = rcutils_get_default_allocator();
     if (RCUTILS_RET_OK !=
-      rcutils_calculate_directory_size(uri.c_str(), &metadata.bag_size, allocator))
+      rcutils_calculate_directory_size_with_recursion(uri.c_str(), 5, &metadata.bag_size, allocator))
     {
       throw std::runtime_error(
               std::string("Exception on calculating the size of directory :") + uri);

rosbag2_storage_default_plugins/CMakeLists.txt

@@ -28,19 +28,32 @@ find_package(rosbag2_storage REQUIRED)
 find_package(sqlite3_vendor REQUIRED)
 find_package(SQLite3 REQUIRED)  # provided by sqlite3_vendor
 find_package(yaml_cpp_vendor REQUIRED)
+find_package(leveldb_vendor REQUIRED)
+find_package(leveldb REQUIRED) # provided by leveldb_vendor
+find_package(Snappy CONFIG QUIET)
 
 add_library(${PROJECT_NAME} SHARED
   src/rosbag2_storage_default_plugins/sqlite/sqlite_wrapper.cpp
   src/rosbag2_storage_default_plugins/sqlite/sqlite_storage.cpp
-  src/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.cpp)
+  src/rosbag2_storage_default_plugins/sqlite/sqlite_statement_wrapper.cpp
+  src/rosbag2_storage_default_plugins/leveldb/leveldb_wrapper.cpp
+  src/rosbag2_storage_default_plugins/leveldb/leveldb_storage.cpp)
+
+if(NOT Snappy_FOUND)
+  set(snappy "")
+else()
+  set(snappy "Snappy")
+endif()
 
 ament_target_dependencies(${PROJECT_NAME}
   pluginlib
   rosbag2_storage
   rcpputils
   rcutils
   SQLite3
-  yaml_cpp_vendor)
+  yaml_cpp_vendor
+  leveldb
+  ${snappy})
 
 target_include_directories(${PROJECT_NAME}
   PUBLIC
@@ -67,7 +80,7 @@ install(
 
 ament_export_include_directories(include)
 ament_export_libraries(${PROJECT_NAME})
-ament_export_dependencies(rosbag2_storage rcpputils rcutils sqlite3_vendor SQLite3)
+ament_export_dependencies(rosbag2_storage rcpputils rcutils sqlite3_vendor SQLite3 leveldb_vendor leveldb)
 
 if(BUILD_TESTING)
   find_package(ament_cmake_gmock REQUIRED)

rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/leveldb/leveldb_exception.hpp

@@ -0,0 +1,45 @@
+// Copyright 2020 Sony Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef ROSBAG2_STORAGE_DEFAULT_PLUGINS__LEVELDB__LEVELDB_EXCEPTION_HPP_
+#define ROSBAG2_STORAGE_DEFAULT_PLUGINS__LEVELDB__LEVELDB_EXCEPTION_HPP_
+
+#include <stdexcept>
+#include <string>
+
+#include "rosbag2_storage_default_plugins/visibility_control.hpp"
+
+// ignore incorrect warning when deriving from standard library types
+#ifdef _WIN32
+# pragma warning(push)
+# pragma warning(disable:4275)
+#endif
+
+namespace rosbag2_storage_plugins
+{
+
+class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC LeveldbException : public std::runtime_error
+{
+public:
+  explicit LeveldbException(const std::string & message)
+  : runtime_error(message) {}
+};
+
+}  // namespace rosbag2_storage_plugins
+
+#ifdef _WIN32
+# pragma warning(pop)
+#endif
+
+#endif  // ROSBAG2_STORAGE_DEFAULT_PLUGINS__LEVELDB__LEVELDB_EXCEPTION_HPP_

rosbag2_storage_default_plugins/include/rosbag2_storage_default_plugins/leveldb/leveldb_storage.hpp

@@ -0,0 +1,112 @@
+// Copyright 2020 Sony Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef ROSBAG2_STORAGE_DEFAULT_PLUGINS__LEVELDB__LEVELDB_STORAGE_HPP_
+#define ROSBAG2_STORAGE_DEFAULT_PLUGINS__LEVELDB__LEVELDB_STORAGE_HPP_
+
+#include <leveldb/db.h>
+#include <leveldb/status.h>
+
+#include <atomic>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <map>
+#include <cassert>
+
+#include "rcutils/types.h"
+
+#include "rosbag2_storage/storage_interfaces/read_write_interface.hpp"
+#include "rosbag2_storage/serialized_bag_message.hpp"
+#include "rosbag2_storage/storage_filter.hpp"
+#include "rosbag2_storage/topic_metadata.hpp"
+
+#include "rosbag2_storage_default_plugins/visibility_control.hpp"
+#include "rosbag2_storage_default_plugins/leveldb/leveldb_wrapper.hpp"
+
+namespace rosbag2_storage_plugins
+{
+
+class ROSBAG2_STORAGE_DEFAULT_PLUGINS_PUBLIC LeveldbStorage
+  : public rosbag2_storage::storage_interfaces::ReadWriteInterface
+{
+public:
+  LeveldbStorage() = default;
+
+  ~LeveldbStorage() override;
+
+  void open(
+    const rosbag2_storage::StorageOptions & storage_options,
+    rosbag2_storage::storage_interfaces::IOFlag io_flag =
+    rosbag2_storage::storage_interfaces::IOFlag::READ_WRITE) override;
+
+  void remove_topic(const rosbag2_storage::TopicMetadata & topic) override;
+
+  void create_topic(const rosbag2_storage::TopicMetadata & topic) override;
+
+  void write(std::shared_ptr<const rosbag2_storage::SerializedBagMessage> message) override;
+
+  void write(
+    const std::vector<std::shared_ptr<const rosbag2_storage::SerializedBagMessage>> & messages)
+  override;
+
+  bool has_next() override;
+
+  std::shared_ptr<rosbag2_storage::SerializedBagMessage> read_next() override;
+
+  std::vector<rosbag2_storage::TopicMetadata> get_all_topics_and_types() override;
+
+  rosbag2_storage::BagMetadata get_metadata() override;
+
+  std::string get_relative_file_path() const override;
+
+  uint64_t get_bagfile_size() const override;
+
+  std::string get_storage_identifier() const override;
+
+  uint64_t get_minimum_split_file_size() const override;
+
+  void set_filter(const rosbag2_storage::StorageFilter & storage_filter) override;
+
+  void reset_filter() override;
+
+private:
+  std::map<std::string, std::shared_ptr<class LeveldbWrapper>> topic_ldb_map_{};
+  std::vector<rosbag2_storage::TopicMetadata> all_topics_and_types_{};
+  std::string relative_path_;
+  rosbag2_storage::StorageFilter storage_filter_{};
+  bool scan_ldb_done_{false};
+
+  struct ts_compare
+  {
+    bool operator()(
+      const rcutils_time_point_value_t & ts1,
+      const rcutils_time_point_value_t & ts2) const
+    {
+      return ts1 < ts2;
+    }
+  };
+  std::map<
+    rcutils_time_point_value_t,
+    std::shared_ptr<rosbag2_storage::SerializedBagMessage>,
+    ts_compare> read_cache_{};
+
+  inline void scan_ldb_for_read();
+  void fill_topics_and_types();
+};
+
+}  // namespace rosbag2_storage_plugins
+
+#endif  // ROSBAG2_STORAGE_DEFAULT_PLUGINS__LEVELDB__LEVELDB_STORAGE_HPP_