From 89620876122314523ef82c3402650977ba2476a1 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 20 Dec 2014 19:33:58 +0100 Subject: [PATCH] replace binarysearcharray with std::vector and std::lower_bound --- Makefile.am | 2 - binarysearcharray.cpp | 118 ------------------------------- binarysearcharray.hpp | 22 ------ node-persistent-cache-reader.cpp | 1 - node-persistent-cache.cpp | 52 +++++++++----- node-persistent-cache.hpp | 32 ++++++++- 6 files changed, 66 insertions(+), 161 deletions(-) delete mode 100644 binarysearcharray.cpp delete mode 100644 binarysearcharray.hpp diff --git a/Makefile.am b/Makefile.am index cd733f787..5e6405ee5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,7 +5,6 @@ bin_PROGRAMS = osm2pgsql nodecachefilereader noinst_LTLIBRARIES = libosm2pgsql.la osm2pgsql_SOURCES = osm2pgsql.cpp \ - binarysearcharray.hpp \ geometry-builder.hpp \ expire-tiles.hpp \ input.hpp \ @@ -39,7 +38,6 @@ osm2pgsql_LDADD = libosm2pgsql.la libosm2pgsql_la_SOURCES = \ UTF8sanitizer.cpp \ - binarysearcharray.cpp \ expire-tiles.cpp \ geometry-builder.cpp \ geometry-processor.cpp \ diff --git a/binarysearcharray.cpp b/binarysearcharray.cpp deleted file mode 100644 index e809c343d..000000000 --- a/binarysearcharray.cpp +++ /dev/null @@ -1,118 +0,0 @@ -#include -#include -#include -#include - -#include "osmtypes.hpp" -#include "binarysearcharray.hpp" - -static int binary_search_lookup(struct binary_search_array * array, int key) -{ - int a = 0; - int b = array->size - 1; - while (a <= b) - { - int pivot = ((b - a) >> 1) + a; - if (array->array[pivot].key == key) - { - return pivot; - } - else if (array->array[pivot].key > key) - { - b = pivot - 1; - } - else - { - a = pivot + 1; - } - } - if ((a < array->size) && (array->array[a].key < key)) - a++; - return a | (1 << (sizeof(int) * 8 - 1)); -} - -osmid_t binary_search_get(struct binary_search_array * array, int key) -{ - int idx; - if (array->size == 0) - return -1; - idx = binary_search_lookup(array, key); - if (idx < 0) - { - return -1; - } - else - { - return array->array[idx].value; - } -} - -void binary_search_remove(struct binary_search_array * array, int key) -{ - int idx = binary_search_lookup(array, key); - if (idx < 0) - { - return; - } - else - { - memmove(&(array->array[idx]), &(array->array[idx + 1]), - sizeof(struct key_val_tuple) * (array->capacity - idx - 1)); - array->size--; - } -} - -void binary_search_add(struct binary_search_array * array, int key, - osmid_t value) -{ - int idx; - if (array->size < array->capacity) - { - if (array->size == 0) - { - array->array[0].key = key; - array->array[0].value = value; - array->size++; - return; - } - idx = binary_search_lookup(array, key); - if (idx < 0) - { - idx = idx & (~(1 << (sizeof(int) * 8 - 1))); - memmove(&(array->array[idx + 1]), &(array->array[idx]), - sizeof(struct key_val_tuple) * (array->capacity - idx - 1)); - array->array[idx].key = key; - array->array[idx].value = value; - array->size++; - } - else - { - fprintf(stderr, "dupplicate!\n"); - exit(1); - } - } -} - -struct binary_search_array * init_search_array(int capacity) -{ - struct binary_search_array * array = - (struct binary_search_array *) - calloc(1, sizeof(struct binary_search_array)); - array->array = (struct key_val_tuple *)calloc(capacity + 1, sizeof(struct key_val_tuple)); - if (!array->array) { - fprintf(stderr, "Out of memory trying to allocate %li bytes for binary search array\n", ((capacity + 1) * sizeof(struct key_val_tuple))); - return NULL; - } - array->capacity = capacity; - array->size = 0; - return array; -} - -void shutdown_search_array(struct binary_search_array ** array) -{ - free((*array)->array); - (*array)->array = NULL; - (*array)->capacity = 0; - free(*array); - *array = NULL; -} diff --git a/binarysearcharray.hpp b/binarysearcharray.hpp deleted file mode 100644 index 8ff9f5b31..000000000 --- a/binarysearcharray.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef BINARYSEARCHARRAY_H -#define BINARYSEARCHARRAY_H - - -struct key_val_tuple { - int key; - osmid_t value; -}; - -struct binary_search_array { - int capacity; - int size; - struct key_val_tuple * array; -}; - -void binary_search_remove(struct binary_search_array * array, int key); -void binary_search_add(struct binary_search_array * array, int key, osmid_t value); -osmid_t binary_search_get(struct binary_search_array * array, int key); -struct binary_search_array * init_search_array(int capacity); -void shutdown_search_array(struct binary_search_array ** array); - -#endif diff --git a/node-persistent-cache-reader.cpp b/node-persistent-cache-reader.cpp index 46938804a..9fb8ec0ad 100644 --- a/node-persistent-cache-reader.cpp +++ b/node-persistent-cache-reader.cpp @@ -16,7 +16,6 @@ #include "options.hpp" #include "node-persistent-cache.hpp" #include "node-ram-cache.hpp" -#include "binarysearcharray.hpp" void test_get_node_list(boost::shared_ptr cache, int itterations, int max_size, int process_number) { diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp index 2c26c7fa8..d70a39928 100644 --- a/node-persistent-cache.cpp +++ b/node-persistent-cache.cpp @@ -17,10 +17,10 @@ #include "output.hpp" #include "options.hpp" #include "node-persistent-cache.hpp" -#include "binarysearcharray.hpp" #include "util.hpp" #include +#include #ifdef _WIN32 #include "win_fsync.h" @@ -171,8 +171,33 @@ int node_persistent_cache::replace_block() */ int node_persistent_cache::find_block(osmid_t block_offset) { - int idx = binary_search_get(readNodeBlockCacheIdx, block_offset); - return idx; + cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(), + readNodeBlockCacheIdx.end(), + block_offset); + if (it != readNodeBlockCacheIdx.end() && it->key == block_offset) + return it->value; + + return -1; +} + +void node_persistent_cache::remove_from_cache_idx(osmid_t block_offset) +{ + cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(), + readNodeBlockCacheIdx.end(), + block_offset); + + if (it == readNodeBlockCacheIdx.end() || it->key != block_offset) + return; + + readNodeBlockCacheIdx.erase(it); +} + +void node_persistent_cache::add_to_cache_idx(cache_index_entry const &entry) +{ + cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(), + readNodeBlockCacheIdx.end(), + entry); + readNodeBlockCacheIdx.insert(it, entry); } /** @@ -232,7 +257,7 @@ void node_persistent_cache::nodes_prefetch_async(osmid_t id) #ifdef HAVE_POSIX_FADVISE osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT; - osmid_t block_id = find_block(block_offset); + int block_id = find_block(block_offset); if (block_id < 0) { /* The needed block isn't in cache already, so initiate loading */ @@ -282,8 +307,7 @@ int node_persistent_cache::load_block(osmid_t block_offset) readNodeBlockCache[block_id].dirty = 0; } - binary_search_remove(readNodeBlockCacheIdx, - readNodeBlockCache[block_id].block_offset); + remove_from_cache_idx(readNodeBlockCache[block_id].block_offset); ramNodes_clear(readNodeBlockCache[block_id].nodes, READ_NODE_BLOCK_SIZE); readNodeBlockCache[block_id].block_offset = block_offset; readNodeBlockCache[block_id].used = READ_NODE_CACHE_SIZE; @@ -311,8 +335,8 @@ int node_persistent_cache::load_block(osmid_t block_offset) strerror(errno)); exit(1); } - binary_search_add(readNodeBlockCacheIdx, - readNodeBlockCache[block_id].block_offset, block_id); + add_to_cache_idx(cache_index_entry(readNodeBlockCache[block_id].block_offset, + block_id)); return block_id; } @@ -461,7 +485,7 @@ int node_persistent_cache::get(struct osmNode *out, osmid_t id) { osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT; - osmid_t block_id = find_block(block_offset); + int block_id = find_block(block_offset); if (block_id < 0) { @@ -564,7 +588,7 @@ int node_persistent_cache::get_list(struct osmNode *nodes, const osmid_t *ndids, node_persistent_cache::node_persistent_cache(const options_t *options, int append, boost::shared_ptr ptr) : node_cache_fd(0), node_cache_fname(NULL), append_mode(0), cacheHeader(), - writeNodeBlock(), readNodeBlockCache(NULL), readNodeBlockCacheIdx(NULL), + writeNodeBlock(), readNodeBlockCache(NULL), scale_(0), cache_already_written(0), ram_cache(ptr) { int i, err; @@ -579,12 +603,7 @@ node_persistent_cache::node_persistent_cache(const options_t *options, int appen fprintf(stderr, "Mid: loading persistent node cache from %s\n", node_cache_fname); - readNodeBlockCacheIdx = init_search_array(READ_NODE_CACHE_SIZE); - if (readNodeBlockCacheIdx == NULL) - { - fprintf(stderr, "Unable to initialise binary search array\n"); - util::exit_nicely(); - } + readNodeBlockCacheIdx.reserve(READ_NODE_CACHE_SIZE); /* Setup the file for the node position cache */ if (append_mode) @@ -746,7 +765,6 @@ node_persistent_cache::~node_persistent_cache() { free(readNodeBlockCache[i].nodes); } - shutdown_search_array(&readNodeBlockCacheIdx); free(readNodeBlockCache); readNodeBlockCache = NULL; } diff --git a/node-persistent-cache.hpp b/node-persistent-cache.hpp index 0c9ee9bb7..11722e895 100644 --- a/node-persistent-cache.hpp +++ b/node-persistent-cache.hpp @@ -4,6 +4,8 @@ #include "node-ram-cache.hpp" #include +#include + #define MAXIMUM_INITIAL_ID 2600000000 #define READ_NODE_CACHE_SIZE 10000 @@ -23,6 +25,29 @@ struct persistentCacheHeader { osmid_t max_initialised_id; }; +struct cache_index_entry { + osmid_t key; + int value; + + cache_index_entry(osmid_t k, int v) : key(k), value(v) {} + cache_index_entry() {} +}; + +inline bool operator<(cache_index_entry const &a, cache_index_entry const &b) +{ + return a.key < b.key; +} + +inline bool operator<(cache_index_entry const &a, osmid_t b) +{ + return a.key < b; +} + +inline bool operator<(osmid_t a, cache_index_entry const &b) +{ + return a < b.key; +} + struct node_persistent_cache : public boost::noncopyable { node_persistent_cache(const struct options_t *options, const int append, @@ -46,6 +71,9 @@ struct node_persistent_cache : public boost::noncopyable int load_block(osmid_t block_offset); void nodes_set_create_writeout_block(); + void remove_from_cache_idx(osmid_t block_offset); + void add_to_cache_idx(cache_index_entry const &entry); + int node_cache_fd; const char * node_cache_fname; int append_mode; @@ -53,7 +81,9 @@ struct node_persistent_cache : public boost::noncopyable struct persistentCacheHeader cacheHeader; struct ramNodeBlock writeNodeBlock; /* larger node block for more efficient initial sequential writing of node cache */ struct ramNodeBlock * readNodeBlockCache; - struct binary_search_array * readNodeBlockCacheIdx; + + typedef std::vector cache_index; + cache_index readNodeBlockCacheIdx; int scale_; int cache_already_written;