From a0641bfb57b6ef6786741b6d8be31a17d91cb3f3 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Wed, 1 Jul 2015 11:31:31 -0700 Subject: [PATCH] Remove the lockfree queue option The boost::lockfree::queue implementation was far less memory efficient than the stack based implementation used with older versions of boost where the lockfree queue was not available. The decreased memory usage allows for more node cache, resulting in faster import times. For the full planet and a 32GB RAM SSD-based server, this is a decrease from 10h45m runtime to 9h6m. Fixes #351 --- Makefile.am | 2 +- configure.ac | 19 ------- m4/ax_boost_atomic.m4 | 119 ------------------------------------------ osmdata.cpp | 33 ------------ output.hpp | 8 +-- 5 files changed, 2 insertions(+), 179 deletions(-) delete mode 100644 m4/ax_boost_atomic.m4 diff --git a/Makefile.am b/Makefile.am index eb4e47b09..8ce3f4f56 100644 --- a/Makefile.am +++ b/Makefile.am @@ -133,7 +133,7 @@ osm2pgsqldir = $(datadir)/osm2pgsql AM_CFLAGS = @PTHREAD_CFLAGS@ @LFS_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ @PROTOBUF_C_CFLAGS@ @ZLIB_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -DVERSION='"@PACKAGE_VERSION@"' @LUA_INCLUDE@ AM_CPPFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -Igeos-fallback @LUA_INCLUDE@ @BOOST_CPPFLAGS@ -GLOBAL_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @XML2_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ @PROTOBUF_C_LDFLAGS@ @PROTOBUF_C_LIBS@ -L/usr/lib/x86_64-linux-gnu @LUA_LIB@ @BOOST_LDFLAGS@ @BOOST_FILESYSTEM_LIB@ @BOOST_SYSTEM_LIB@ @BOOST_THREAD_LIB@ @BOOST_ATOMIC_LIB@ +GLOBAL_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @XML2_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ @PROTOBUF_C_LDFLAGS@ @PROTOBUF_C_LIBS@ -L/usr/lib/x86_64-linux-gnu @LUA_LIB@ @BOOST_LDFLAGS@ @BOOST_FILESYSTEM_LIB@ @BOOST_SYSTEM_LIB@ @BOOST_THREAD_LIB@ osm2pgsql_LDADD += $(GLOBAL_LDFLAGS) tests_test_parse_xml2_LDADD += $(GLOBAL_LDFLAGS) tests_test_middle_ram_LDADD += $(GLOBAL_LDFLAGS) diff --git a/configure.ac b/configure.ac index 045a22f22..26c81082f 100644 --- a/configure.ac +++ b/configure.ac @@ -117,25 +117,6 @@ then AC_MSG_ERROR([One or more of the mandatory Boost libraries not found.]) fi -dnl Check if Boost is recent enough for lockfree, and if it hasn't been overridden -AC_ARG_WITH([lockfree], - [AS_HELP_STRING([--without-lockfree], - [disable lockfree queue])], - [], - [with_lockfree=yes]) - -if test "x$with_lockfree" = "xyes" -then - AX_BOOST_BASE([1.53], - [ - AX_BOOST_ATOMIC - if test "x$BOOST_ATOMIC_LIB" != "x"; then - AC_DEFINE([HAVE_LOCKFREE], [1], [Using lockfree queue]) - fi - ] - , []) -fi - dnl Boost json parser in 1.49 has a bug when compiled with C++11 dnl see https://svn.boost.org/trac/boost/ticket/6785 AC_ARG_WITH([cxx11], diff --git a/m4/ax_boost_atomic.m4 b/m4/ax_boost_atomic.m4 deleted file mode 100644 index f918f31fa..000000000 --- a/m4/ax_boost_atomic.m4 +++ /dev/null @@ -1,119 +0,0 @@ -# SYNOPSIS -# -# AX_BOOST_ATOMIC -# -# DESCRIPTION -# -# Test for Atomic library from the Boost C++ libraries. The macro requires -# a preceding call to AX_BOOST_BASE. Further documentation is available at -# . -# -# This macro calls: -# -# AC_SUBST(BOOST_ATOMIC_LIB) -# -# And sets: -# -# HAVE_BOOST_ATOMIC - -# -# LICENSE -# -# Copyright (c) 2015 Sarah Hoffmann -# Copyright (c) 2008 Thomas Porschberg -# Copyright (c) 2008 Michael Tindal -# Copyright (c) 2008 Daniel Casimiro -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 1 - -AC_DEFUN([AX_BOOST_ATOMIC], -[ - AC_ARG_WITH([boost-atomic], - AS_HELP_STRING([--with-boost-atomic@<:@=special-lib@:>@], - [use the Atomic library from boost - it is possible to specify a certain library for the linker - e.g. --with-boost-atomic=boost_atomic-gcc-mt ]), - [ - if test "$withval" = "no"; then - want_boost="no" - elif test "$withval" = "yes"; then - want_boost="yes" - ax_boost_user_atomic_lib="" - else - want_boost="yes" - ax_boost_user_atomic_lib="$withval" - fi - ], - [want_boost="yes"] - ) - - if test "x$want_boost" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - AC_REQUIRE([AC_CANONICAL_BUILD]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - AC_CACHE_CHECK(whether the Boost::Atomic library is available, - ax_cv_boost_atomic, - [AC_LANG_PUSH([C++]) - CXXFLAGS_SAVE=$CXXFLAGS - - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[boost::atomic_size_t x;]])], - ax_cv_boost_atomic=yes, ax_cv_boost_atomic=no) - CXXFLAGS=$CXXFLAGS_SAVE - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_atomic" = "xyes"; then - AC_SUBST(BOOST_CPPFLAGS) - - AC_DEFINE(HAVE_BOOST_ATOMIC,,[define if the Boost::Atomic library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - - LDFLAGS_SAVE=$LDFLAGS - if test "x$ax_boost_user_atomic_lib" = "x"; then - for libextension in `ls -r $BOOSTLIBDIR/libboost_atomic* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_ATOMIC_LIB="-l$ax_lib"; AC_SUBST(BOOST_ATOMIC_LIB) link_atomic="yes"; break], - [link_atomic="no"]) - done - if test "x$link_atomic" != "xyes"; then - for libextension in `ls -r $BOOSTLIBDIR/boost_atomic* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_ATOMIC_LIB="-l$ax_lib"; AC_SUBST(BOOST_ATOMIC_LIB) link_atomic="yes"; break], - [link_atomic="no"]) - done - fi - - else - for ax_lib in $ax_boost_user_atomic_lib boost_atomic-$ax_boost_user_atomic_lib; do - AC_CHECK_LIB($ax_lib, exit, - [BOOST_ATOMIC_LIB="-l$ax_lib"; AC_SUBST(BOOST_ATOMIC_LIB) link_atomic="yes"; break], - [link_atomic="no"]) - done - - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the library!) - fi - if test "x$link_atomic" = "xno"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - fi -]) - diff --git a/osmdata.cpp b/osmdata.cpp index 7d71734a9..95c25b54a 100644 --- a/osmdata.cpp +++ b/osmdata.cpp @@ -14,10 +14,6 @@ #include #include -#ifdef HAVE_LOCKFREE -#include -#endif - osmdata_t::osmdata_t(boost::shared_ptr mid_, const boost::shared_ptr& out_): mid(mid_) { outs.push_back(out_); @@ -176,7 +172,6 @@ struct pending_threaded_processor : public middle_t::pending_processor { typedef std::vector > output_vec_t; typedef std::pair, output_vec_t> clone_t; -#ifndef HAVE_LOCKFREE static void do_jobs(output_vec_t const& outputs, pending_queue_t& queue, size_t& ids_done, boost::mutex& mutex, int append, bool ways) { while (true) { //get the job off the queue synchronously @@ -203,29 +198,13 @@ struct pending_threaded_processor : public middle_t::pending_processor { mutex.unlock(); } } -#else - static void do_jobs(output_vec_t const& outputs, pending_queue_t& queue, boost::atomic_size_t& ids_done, int append, bool ways) { - pending_job_t job; - while (queue.pop(job)) { - if(ways) - outputs.at(job.output_id)->pending_way(job.osm_id, append); - else - outputs.at(job.output_id)->pending_relation(job.osm_id, append); - ++ids_done; - } - } -#endif //starts up count threads and works on the queue pending_threaded_processor(boost::shared_ptr mid, const output_vec_t& outs, size_t thread_count, size_t job_count, int append) -#ifndef HAVE_LOCKFREE //note that we cant hint to the stack how large it should be ahead of time //we could use a different datastructure like a deque or vector but then //the outputs the enqueue jobs would need the version check for the push(_back) method : outs(outs), ids_queued(0), append(append), queue(), ids_done(0) { -#else - : outs(outs), ids_queued(0), append(append), queue(job_count), ids_done(0) { -#endif //clone all the things we need clones.reserve(thread_count); @@ -265,11 +244,7 @@ struct pending_threaded_processor : public middle_t::pending_processor { //make the threads and start them for (size_t i = 0; i < clones.size(); ++i) { -#ifndef HAVE_LOCKFREE workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), boost::ref(mutex), append, true)); -#else - workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), append, true)); -#endif } //TODO: print out partial progress @@ -316,11 +291,7 @@ struct pending_threaded_processor : public middle_t::pending_processor { //make the threads and start them for (size_t i = 0; i < clones.size(); ++i) { -#ifndef HAVE_LOCKFREE workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), boost::ref(mutex), append, false)); -#else - workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), append, false)); -#endif } //TODO: print out partial progress @@ -362,14 +333,10 @@ struct pending_threaded_processor : public middle_t::pending_processor { //job queue pending_queue_t queue; -#ifndef HAVE_LOCKFREE //how many ids within the job have been processed size_t ids_done; //so the threads can manage some of the shared state boost::mutex mutex; -#else - boost::atomic_size_t ids_done; -#endif }; } // anonymous namespace diff --git a/output.hpp b/output.hpp index 4f8b8144e..bf93f5d6a 100644 --- a/output.hpp +++ b/output.hpp @@ -18,7 +18,7 @@ #include #include #include - +#include struct pending_job_t { osmid_t osm_id; @@ -28,13 +28,7 @@ struct pending_job_t { pending_job_t(osmid_t id, size_t oid) : osm_id(id), output_id(oid) {} }; -#ifndef HAVE_LOCKFREE -#include typedef std::stack pending_queue_t; -#else -#include -typedef boost::lockfree::queue pending_queue_t; -#endif class output_t : public boost::noncopyable { public: