From 77c5707a58bce849053c0eb7773523d05c5ac5a5 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Sun, 22 Feb 2015 10:10:20 +0000 Subject: [PATCH] Removed all references to LevelDB - it has not been the recommended disk database format for a while now. --- README.md | 7 ++--- configure.ac | 14 --------- include/copy_elements.hpp | 2 +- include/insert_kv.hpp | 5 --- src/Makefile.am | 5 --- src/copy_elements.cpp | 52 ------------------------------- src/dump_archive.cpp | 2 +- src/dump_reader.cpp | 65 --------------------------------------- src/insert_kv.cpp | 4 --- src/planet-dump.cpp | 6 ++-- 10 files changed, 7 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index 8a8b025..6d6f0a3 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,7 @@ Before building the code, you will need: * libxml2 (version 2.6.31 recommended), * The Boost libraries (version 1.49 recommended), * libosmpbf (version 1.3.0 recommended), -* leveldb (version 1.9.0 recommended), * libprotobuf and libprotobuf-lite (version 2.4.1 recommended) -* libsnappy To install these on Ubuntu, you can just type: @@ -30,8 +28,7 @@ To install these on Ubuntu, you can just type: libxml2-dev libboost-dev libboost-program-options-dev \ libboost-date-time-dev libboost-filesystem-dev \ libboost-thread-dev libboost-iostreams-dev \ - libosmpbf-dev osmpbf-bin libsnappy-dev \ - libprotobuf-dev pkg-config + libosmpbf-dev osmpbf-bin libprotobuf-dev pkg-config After that, it should just be a matter of running: @@ -51,7 +48,7 @@ you can read by running: planet-dump-ng --help -One thing to note is that the program will create LevelDB databases in +One thing to note is that the program will create on-disk databases in the current working directory, so it is wise to run the program somewhere with plenty of fast disk space. Existing files may interfere with the operation of the program, so it's best to run it in its own, diff --git a/configure.ac b/configure.ac index 8c62bc3..84a53d7 100644 --- a/configure.ac +++ b/configure.ac @@ -26,20 +26,6 @@ AX_BOOST_DATE_TIME AX_BOOST_THREAD AX_BOOST_IOSTREAMS -AC_ARG_WITH(leveldb, - [AS_HELP_STRING([--with-leveldb=], [Path containing LevelDB library (e.g: /usr), if you want to use it.])], - [LEVELDB_PREFIX=$with_leveldb]) -AS_IF([test "x$LEVELDB_PREFIX" != x], - [AC_DEFINE([HAVE_LEVELDB], [1], [Define when LevelDB library is to be used.])], - []) -AM_CONDITIONAL([HAVE_LEVELDB], [test "x$LEVELDB_PREFIX" != x]) - -AC_SUBST(LEVELDB_PREFIX) -LEVELDB_LIBS="-L${LEVELDB_PREFIX}/lib -lleveldb -lsnappy" -LEVELDB_CFLAGS="-I${LEVELDB_PREFIX}/include" -AC_SUBST(LEVELDB_LIBS) -AC_SUBST(LEVELDB_CFLAGS) - PKG_CHECK_MODULES([PROTOBUF_LITE], "protobuf-lite") AC_SUBST([PROTOBUF_LITE_CFLAGS]) AC_SUBST([PROTOBUF_LITE_LIBS]) diff --git a/include/copy_elements.hpp b/include/copy_elements.hpp index 64a635f..d95cf9e 100644 --- a/include/copy_elements.hpp +++ b/include/copy_elements.hpp @@ -7,7 +7,7 @@ #include /** - * Read the LevelDB database for users, and extract all the public data + * Read the disk database for users, and extract all the public data * ones into a map of user ID to display name. */ void extract_users(std::map &display_name_map); diff --git a/include/insert_kv.hpp b/include/insert_kv.hpp index ec84d69..a295078 100644 --- a/include/insert_kv.hpp +++ b/include/insert_kv.hpp @@ -3,13 +3,8 @@ #include "config.h" -#ifdef HAVE_LEVELDB -#include -typedef leveldb::Slice slice_t; -#else /* HAVE_LEVELDB */ #include typedef std::string slice_t; -#endif /* HAVE_LEVELDB */ template void insert_kv(T &t, const slice_t &key, const slice_t &val); diff --git a/src/Makefile.am b/src/Makefile.am index 28e8e90..56b62bc 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -3,11 +3,6 @@ LDADD=@LIBXML_LIBS@ @BOOST_FILESYSTEM_LIB@ @BOOST_PROGRAM_OPTIONS_LIB@ @BOOST_DA AM_LDFLAGS=@BOOST_LDFLAGS@ AM_CPPFLAGS=-I../include @LIBXML_CFLAGS@ @BOOST_CPPFLAGS@ @PROTOBUF_LITE_CFLAGS@ @PROTOBUF_CFLAGS@ -if HAVE_LEVELDB -LDADD+=@LEVELDB_LIBS@ -AM_CPPFLAGS+=@LEVELDB_CFLAGS@ -endif - bin_PROGRAMS=../planet-dump-ng ################################################################################ ___planet_dump_ng_SOURCES=\ diff --git a/src/copy_elements.cpp b/src/copy_elements.cpp index 19c3442..3cb4188 100644 --- a/src/copy_elements.cpp +++ b/src/copy_elements.cpp @@ -18,22 +18,15 @@ #include #include -#ifdef HAVE_LEVELDB -#include -#include -#else /* HAVE_LEVELDB */ #include #include #include #include #include #include -#endif /* HAVE_LEVELDB */ -#ifndef HAVE_LEVELDB namespace bio = boost::iostreams; namespace fs = boost::filesystem; -#endif /* HAVE_LEVELDB */ namespace { @@ -78,50 +71,6 @@ struct thread_writer { } }; -#ifdef HAVE_LEVELDB -// leveldb implementation of db_reader -template -struct db_reader { - db_reader(const std::string &name) : m_db(NULL), m_itr(NULL) { - m_options.create_if_missing = false; - m_options.error_if_exists = false; - - leveldb::Status status; - status = leveldb::DB::Open(m_options, name, &m_db); - if (!status.ok()) { - BOOST_THROW_EXCEPTION(std::runtime_error((boost::format("Can't open database: %1%") % status.ToString()).str())); - } - - m_itr = m_db->NewIterator(m_read_options); - m_itr->SeekToFirst(); - } - - ~db_reader() { - if (m_itr != NULL) { - delete m_itr; - } - if (m_db != NULL) { - delete m_db; - } - } - - bool operator()(T &t) { - const bool valid = m_itr->Valid(); - if (valid) { - leveldb::Slice key = m_itr->key(); - leveldb::Slice val = m_itr->value(); - insert_kv(t, key, val); - m_itr->Next(); - } - return valid; - } - - leveldb::DB *m_db; - leveldb::Iterator *m_itr; - leveldb::Options m_options; - leveldb::ReadOptions m_read_options; -}; -#else /* HAVE_LEVELDB */ template struct db_reader { explicit db_reader(const std::string &subdir) : m_end(false) { @@ -169,7 +118,6 @@ struct db_reader { std::ifstream m_file; bio::filtering_streambuf m_stream; }; -#endif /* HAVE_LEVELDB */ template <> struct db_reader { diff --git a/src/dump_archive.cpp b/src/dump_archive.cpp index 53f73bc..75bd646 100644 --- a/src/dump_archive.cpp +++ b/src/dump_archive.cpp @@ -107,7 +107,7 @@ template bt::ptime run_thread::join() { thr.join(); if (error) { - boost::throw_exception(boost::enable_error_info(std::runtime_error("Error during archive dump to LevelDB.")) + boost::throw_exception(boost::enable_error_info(std::runtime_error("Error during archive dump to disk database.")) << boost::errinfo_nested_exception(error) << errinfo_table_name(table_name)); } diff --git a/src/dump_reader.cpp b/src/dump_reader.cpp index 0745092..34c0639 100644 --- a/src/dump_reader.cpp +++ b/src/dump_reader.cpp @@ -9,11 +9,6 @@ #include #include -#ifdef HAVE_LEVELDB -#include -#include -#include -#else /* HAVE_LEVELDB */ #include #include #include @@ -23,7 +18,6 @@ #include #include //#include -#endif /* HAVE_LEVELDB */ #include #include @@ -38,22 +32,17 @@ namespace { namespace qi = boost::spirit::qi; -#ifndef HAVE_LEVELDB namespace bio = boost::iostreams; namespace fs = boost::filesystem; -#endif /* !HAVE_LEVELDB */ struct tag_copy_header; -struct tag_leveldb_status; typedef boost::error_info copy_header; -typedef boost::error_info leveldb_status; struct popen_error : public boost::exception, std::exception {}; struct fread_error : public boost::exception, std::exception {}; struct early_termination_error : public boost::exception, std::exception {}; struct copy_header_parse_error : public boost::exception, std::exception {}; -struct leveldb_error : public boost::exception, std::exception {}; typedef boost::shared_ptr pipe_ptr; @@ -245,59 +234,6 @@ struct filter_copy_contents std::string m_table_name; }; -#ifdef HAVE_LEVELDB -struct db_writer { - explicit db_writer(const std::string &table_name) - : m_db(NULL), - m_batch(), - m_batch_size(0), - m_write_options() { - - leveldb::Options options; - options.create_if_missing = true; - options.error_if_exists = true; - - // bigger write buffer, as this is a write-heavy process... - options.write_buffer_size = 128 * 1024 * 1024; - - leveldb::Status status = leveldb::DB::Open(options, table_name, &m_db); - if (!status.ok()) { - BOOST_THROW_EXCEPTION(leveldb_error() << leveldb_status(status.ToString())); - } - } - - ~db_writer() { - delete m_db; - } - - void finish() { - if (m_batch_size > 0) { - m_db->Write(m_write_options, &m_batch); - m_batch.Clear(); - m_batch_size = 0; - } - m_db->CompactRange(NULL, NULL); - } - - void put(const std::string &k, const std::string &v) { - m_batch.Put(k, v); - ++m_batch_size; - - if (m_batch_size >= BATCH_SIZE) { - m_db->Write(m_write_options, &m_batch); - m_batch.Clear(); - m_batch_size = 0; - } - } - - leveldb::DB *m_db; - leveldb::WriteBatch m_batch; - size_t m_batch_size; - leveldb::WriteOptions m_write_options; -}; - -#else /* HAVE_LEVELDB */ - typedef std::pair kv_pair_t; struct block_reader : public boost::noncopyable { @@ -607,7 +543,6 @@ struct db_writer : public boost::noncopyable { if (tcb.m_error) { boost::rethrow_exception(tcb.m_error); } } }; -#endif /* HAVE_LEVELDB */ } // anonymous namespace diff --git a/src/insert_kv.cpp b/src/insert_kv.cpp index c7270c4..6b455af 100644 --- a/src/insert_kv.cpp +++ b/src/insert_kv.cpp @@ -139,11 +139,7 @@ struct unapp_item { template void from_binary(const slice_t &s, T &t) { -#ifdef HAVE_LEVELDB - std::istringstream in(s.ToString()); -#else /* HAVE_LEVELDB */ std::istringstream in(s); -#endif /* HAVE_LEVELDB */ bf::fold(t, 0, unapp_item(in)); } diff --git a/src/planet-dump.cpp b/src/planet-dump.cpp index cbb64ec..7ce7082 100644 --- a/src/planet-dump.cpp +++ b/src/planet-dump.cpp @@ -69,12 +69,12 @@ static void get_options(int argc, char **argv, po::variables_map &vm) { } /** - * read the dump file in parallel to get all of the elements into leveldb + * read the dump file in parallel to get all of the elements into on-disk * databases. this is primarily so that the data is sorted, which is not * guaranteed in the PostgreSQL dump file. returns the maximum time seen * in a timestamp of any element in the dump file. */ -bt::ptime setup_leveldb_databases(const std::string &dump_file) { +bt::ptime setup_databases(const std::string &dump_file) { std::list > threads; threads.push_back(boost::make_shared >("changesets", dump_file)); @@ -113,7 +113,7 @@ int main(int argc, char *argv[]) { // extract data from the dump file for the "sorted" data tables, like nodes, // ways, relations, changesets and their associated tags, etc... const std::string dump_file(options["dump-file"].as()); - const bt::ptime max_time = setup_leveldb_databases(dump_file); + const bt::ptime max_time = setup_databases(dump_file); // users aren't dumped directly to the files. we only use them to build up a map // of uid -> name where a missing uid indicates that the user doesn't have public