From c95ddbf9029aac442b1ef73d5c158098b3f509a1 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 3 Jun 2015 22:48:59 +0200 Subject: [PATCH 1/6] allocate cache for flatnodes only when required --- node-persistent-cache.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp index 61a4804ae..66f3505ed 100644 --- a/node-persistent-cache.cpp +++ b/node-persistent-cache.cpp @@ -235,8 +235,16 @@ int node_persistent_cache::load_block(osmid_t block_offset) readNodeBlockCache[block_id].dirty = 0; } - remove_from_cache_idx(readNodeBlockCache[block_id].block_offset); - new(readNodeBlockCache[block_id].nodes) ramNode[READ_NODE_BLOCK_SIZE]; + if (readNodeBlockCache[block_id].nodes) { + remove_from_cache_idx(readNodeBlockCache[block_id].block_offset); + new(readNodeBlockCache[block_id].nodes) ramNode[READ_NODE_BLOCK_SIZE]; + } else { + readNodeBlockCache[block_id].nodes = new ramNode[READ_NODE_BLOCK_SIZE]; + if (!readNodeBlockCache[block_id].nodes) { + fprintf(stderr, "Out of memory: Failed to allocate node read cache\n"); + util::exit_nicely(); + } + } readNodeBlockCache[block_id].block_offset = block_offset; readNodeBlockCache[block_id].used = READ_NODE_CACHE_SIZE; @@ -613,11 +621,7 @@ node_persistent_cache::node_persistent_cache(const options_t *options, int appen } for (i = 0; i < READ_NODE_CACHE_SIZE; i++) { - readNodeBlockCache[i].nodes = new ramNode[READ_NODE_BLOCK_SIZE]; - if (!readNodeBlockCache[i].nodes) { - fprintf(stderr, "Out of memory: Failed to allocate node read cache\n"); - util::exit_nicely(); - } + readNodeBlockCache[i].nodes = 0; readNodeBlockCache[i].block_offset = -1; readNodeBlockCache[i].used = 0; readNodeBlockCache[i].dirty = 0; @@ -659,7 +663,8 @@ node_persistent_cache::~node_persistent_cache() if (readNodeBlockCache) { for (int i = 0; i < READ_NODE_CACHE_SIZE; i++) { - delete[] readNodeBlockCache[i].nodes; + if (readNodeBlockCache[i].nodes) + delete[] readNodeBlockCache[i].nodes; } delete[] readNodeBlockCache; } From c26e969eecf3ee968b0f3e7e0b18da1f9692e295 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 4 Jun 2015 00:17:36 +0200 Subject: [PATCH 2/6] use a more condense structure for ramNodeBlock --- node-persistent-cache.cpp | 53 ++++++++++++++++----------------------- node-ram-cache.cpp | 22 ++++++++-------- node-ram-cache.hpp | 20 ++++++++++++--- 3 files changed, 48 insertions(+), 47 deletions(-) diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp index 66f3505ed..43243d9ba 100644 --- a/node-persistent-cache.cpp +++ b/node-persistent-cache.cpp @@ -49,7 +49,7 @@ void node_persistent_cache::writeout_dirty_nodes() { for (int i = 0; i < READ_NODE_CACHE_SIZE; i++) { - if (readNodeBlockCache[i].dirty) + if (readNodeBlockCache[i].dirty()) { if (lseek64(node_cache_fd, (readNodeBlockCache[i].block_offset @@ -70,7 +70,7 @@ void node_persistent_cache::writeout_dirty_nodes() util::exit_nicely(); } } - readNodeBlockCache[i].dirty = 0; + readNodeBlockCache[i].reset_used(); } } @@ -85,9 +85,9 @@ int node_persistent_cache::replace_block() for (int i = 0; i < READ_NODE_CACHE_SIZE; i++) { - if (readNodeBlockCache[i].used < min_used) + if (readNodeBlockCache[i].used() < min_used) { - min_used = readNodeBlockCache[i].used; + min_used = readNodeBlockCache[i].used(); block_id = i; } } @@ -95,9 +95,9 @@ int node_persistent_cache::replace_block() { for (int i = 0; i < READ_NODE_CACHE_SIZE; i++) { - if (readNodeBlockCache[i].used > 1) + if (readNodeBlockCache[i].used() > 1) { - readNodeBlockCache[i].used--; + readNodeBlockCache[i].dec_used(); } } } @@ -214,7 +214,7 @@ int node_persistent_cache::load_block(osmid_t block_offset) { const int block_id = replace_block(); - if (readNodeBlockCache[block_id].dirty) + if (readNodeBlockCache[block_id].dirty()) { if (lseek64(node_cache_fd, (readNodeBlockCache[block_id].block_offset @@ -232,7 +232,7 @@ int node_persistent_cache::load_block(osmid_t block_offset) strerror(errno)); util::exit_nicely(); } - readNodeBlockCache[block_id].dirty = 0; + readNodeBlockCache[block_id].reset_used(); } if (readNodeBlockCache[block_id].nodes) { @@ -246,7 +246,7 @@ int node_persistent_cache::load_block(osmid_t block_offset) } } readNodeBlockCache[block_id].block_offset = block_offset; - readNodeBlockCache[block_id].used = READ_NODE_CACHE_SIZE; + readNodeBlockCache[block_id].set_used(READ_NODE_CACHE_SIZE); /* Make sure the node cache is correctly initialised for the block that will be read */ if (cacheHeader.max_initialised_id @@ -328,14 +328,13 @@ int node_persistent_cache::set_create(osmid_t id, double lat, double lon) assert(!append_mode); assert(!read_mode); - osmid_t block_offset = id >> WRITE_NODE_BLOCK_SHIFT; + int block_offset = id >> WRITE_NODE_BLOCK_SHIFT; if (writeNodeBlock.block_offset != block_offset) { - if (writeNodeBlock.dirty) + if (writeNodeBlock.dirty()) { nodes_set_create_writeout_block(); - writeNodeBlock.dirty = 0; /* After writing out the node block, the file pointer is at the next block level */ writeNodeBlock.block_offset++; cacheHeader.max_initialised_id = (writeNodeBlock.block_offset @@ -344,7 +343,7 @@ int node_persistent_cache::set_create(osmid_t id, double lat, double lon) if (writeNodeBlock.block_offset > block_offset) { fprintf(stderr, - "ERROR: Block_offset not in sequential order: %" PRIdOSMID "%" PRIdOSMID "\n", + "ERROR: Block_offset not in sequential order: %d %d\n", writeNodeBlock.block_offset, block_offset); util::exit_nicely(); } @@ -361,7 +360,7 @@ int node_persistent_cache::set_create(osmid_t id, double lat, double lon) } writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK] = ramNode(lon, lat); - writeNodeBlock.dirty = 1; + writeNodeBlock.set_dirty(); return 0; } @@ -381,8 +380,8 @@ int node_persistent_cache::set_append(osmid_t id, double lat, double lon) readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK] = ramNode(); else readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK] = ramNode(lon, lat); - readNodeBlockCache[block_id].used++; - readNodeBlockCache[block_id].dirty = 1; + readNodeBlockCache[block_id].inc_used(); + readNodeBlockCache[block_id].set_dirty(); return 1; } @@ -407,7 +406,7 @@ int node_persistent_cache::get(osmNode *out, osmid_t id) block_id = load_block(block_offset); } - readNodeBlockCache[block_id].used++; + readNodeBlockCache[block_id].inc_used(); if (!readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].is_valid()) return 1; @@ -459,11 +458,11 @@ void node_persistent_cache::set_read_mode() if (read_mode) return; - if (writeNodeBlock.dirty > 0) { + if (writeNodeBlock.dirty()) { assert(!append_mode); fprintf(stderr, "Switching to read mode\n"); nodes_set_create_writeout_block(); - writeNodeBlock.dirty = 0; + writeNodeBlock.reset_used(); writeNodeBlock.block_offset++; cacheHeader.max_initialised_id = (writeNodeBlock.block_offset << WRITE_NODE_BLOCK_SHIFT) - 1; @@ -492,7 +491,6 @@ node_persistent_cache::node_persistent_cache(const options_t *options, int appen : node_cache_fd(0), node_cache_fname(NULL), append_mode(0), cacheHeader(), writeNodeBlock(), readNodeBlockCache(NULL), read_mode(ro), ram_cache(ptr) { - int i, err; append_mode = append; if (options->flat_node_file) { node_cache_fname = options->flat_node_file->c_str(); @@ -541,12 +539,10 @@ node_persistent_cache::node_persistent_cache(const options_t *options, int appen }; writeNodeBlock.block_offset = 0; - writeNodeBlock.dirty = 0; - writeNodeBlock.nodes = 0; if (!read_mode) { - + int err; #ifdef HAVE_POSIX_FALLOCATE if ((err = posix_fallocate(node_cache_fd, 0, sizeof(ramNode) * MAXIMUM_INITIAL_ID)) != 0) @@ -614,23 +610,16 @@ node_persistent_cache::node_persistent_cache(const options_t *options, int appen fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id); - readNodeBlockCache = new ramNodeBlock [READ_NODE_CACHE_SIZE]; + readNodeBlockCache = new ramNodeBlock[READ_NODE_CACHE_SIZE]; if (!readNodeBlockCache) { fprintf(stderr, "Out of memory: Failed to allocate node read cache\n"); util::exit_nicely(); } - for (i = 0; i < READ_NODE_CACHE_SIZE; i++) - { - readNodeBlockCache[i].nodes = 0; - readNodeBlockCache[i].block_offset = -1; - readNodeBlockCache[i].used = 0; - readNodeBlockCache[i].dirty = 0; - } } node_persistent_cache::~node_persistent_cache() { - if (writeNodeBlock.dirty > 0) + if (writeNodeBlock.dirty()) nodes_set_create_writeout_block(); writeout_dirty_nodes(); diff --git a/node-ram-cache.cpp b/node-ram-cache.cpp index 167f6b775..67fe5e25a 100644 --- a/node-ram-cache.cpp +++ b/node-ram-cache.cpp @@ -84,7 +84,7 @@ void node_ram_cache::percolate_up( int pos ) while( i > 0 ) { int parent = (i-1)>>1; - if( queue[i]->used < queue[parent]->used ) + if( queue[i]->used() < queue[parent]->used() ) { Swap( queue[i], queue[parent] ) i = parent; @@ -143,7 +143,7 @@ int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { * to store it in dense representation. If not, push all elements of the block * to the sparse node cache and reuse memory of the previous block for the current block */ if ( ((allocStrategy & ALLOC_SPARSE) == 0) || - ((queue[usedBlocks - 1]->used / (double)(1<< BLOCK_SHIFT)) > + ((queue[usedBlocks - 1]->used() / (double)(1<< BLOCK_SHIFT)) > (sizeof(ramNode) / (double)sizeof(ramNodeID)))) { /* Block has reached the level to keep it in dense representation */ /* We've just finished with the previous block, so we need to percolate it up the queue to its correct position */ @@ -160,7 +160,7 @@ int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { } } /* reuse previous block, as its content is now in the sparse representation */ - storedNodes -= queue[usedBlocks - 1]->used; + storedNodes -= queue[usedBlocks - 1]->used(); blocks[block].nodes = queue[usedBlocks - 1]->nodes; blocks[queue[usedBlocks - 1]->block_offset].nodes = NULL; usedBlocks--; @@ -170,7 +170,7 @@ int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { blocks[block].nodes = next_chunk(); } - blocks[block].used = 0; + blocks[block].reset_used(); blocks[block].block_offset = block; if (!blocks[block].nodes) { fprintf(stderr, "Error allocating nodes\n"); @@ -196,15 +196,15 @@ int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { i=0; while( 2*i+1 < usedBlocks - 1 ) { - if( queue[2*i+1]->used <= queue[2*i+2]->used ) { - if( queue[i]->used > queue[2*i+1]->used ) { + if( queue[2*i+1]->used() <= queue[2*i+2]->used() ) { + if( queue[i]->used() > queue[2*i+1]->used() ) { Swap( queue[i], queue[2*i+1] ); i = 2*i+1; } else break; } else { - if( queue[i]->used > queue[2*i+2]->used ) { + if( queue[i]->used() > queue[2*i+2]->used() ) { Swap( queue[i], queue[2*i+2] ); i = 2*i+2; } else @@ -213,13 +213,13 @@ int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { } /* Now the head of the queue is the smallest, so it becomes our replacement candidate */ blocks[block].nodes = queue[0]->nodes; - blocks[block].used = 0; + blocks[block].reset_used(); new(blocks[block].nodes) ramNode[PER_BLOCK]; /* Clear old head block and point to new block */ - storedNodes -= queue[0]->used; + storedNodes -= queue[0]->used(); queue[0]->nodes = NULL; - queue[0]->used = 0; + queue[0]->reset_used(); queue[0] = &blocks[block]; } } else { @@ -243,7 +243,7 @@ int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { } blocks[block].nodes[offset] = coord; - blocks[block].used++; + blocks[block].inc_used(); storedNodes++; return 0; } diff --git a/node-ram-cache.hpp b/node-ram-cache.hpp index 04049047d..13bc8d75f 100644 --- a/node-ram-cache.hpp +++ b/node-ram-cache.hpp @@ -83,11 +83,23 @@ struct ramNodeID { ramNode coord; }; -struct ramNodeBlock { +class ramNodeBlock { +public: + ramNodeBlock() : nodes(NULL), block_offset(-1), _used(0) {} + + void set_dirty() { _used |= 1; } + bool dirty() const { return _used & 1; } + + void reset_used() { _used = 0; } + void inc_used() { _used += 2; } + void dec_used() { _used -= 2; } + void set_used(int used) { _used = (used << 1) || (_used & 1); } + int used() const { return _used >> 1; } + ramNode *nodes; - osmid_t block_offset; - int used; - int dirty; + int block_offset; +private: + int _used; // 0-bit indicates dirty }; struct node_ram_cache : public boost::noncopyable From 1a6c245a2cf7e02018d68d7eb384eb0e1f31af54 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 16 Jun 2015 21:18:17 +0200 Subject: [PATCH 3/6] enforce ordered nodes in sparse cache fixes #151 --- node-ram-cache.cpp | 9 +++++++-- node-ram-cache.hpp | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/node-ram-cache.cpp b/node-ram-cache.cpp index 67fe5e25a..b349bf21f 100644 --- a/node-ram-cache.cpp +++ b/node-ram-cache.cpp @@ -108,7 +108,11 @@ ramNode *node_ram_cache::next_chunk() { int node_ram_cache::set_sparse(osmid_t id, const ramNode &coord) { - if ((sizeSparseTuples > maxSparseTuples) || ( cacheUsed > cacheSize)) { + // Sparse cache depends on ordered nodes, reject out-of-order ids. + // Also check that there is still space. + if ((maxSparseId && id < maxSparseId) + || (sizeSparseTuples > maxSparseTuples) + || ( cacheUsed > cacheSize)) { if ((allocStrategy & ALLOC_LOSSY) > 0) return 1; else { @@ -116,6 +120,7 @@ int node_ram_cache::set_sparse(osmid_t id, const ramNode &coord) { util::exit_nicely(); } } + maxSparseId = id; sparseBlock[sizeSparseTuples].id = id; sparseBlock[sizeSparseTuples].coord = coord; @@ -294,7 +299,7 @@ int node_ram_cache::get_dense(osmNode *out, osmid_t id) { node_ram_cache::node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale ) : allocStrategy(ALLOC_DENSE), blocks(NULL), usedBlocks(0), maxBlocks(0), blockCache(NULL), queue(NULL), sparseBlock(NULL), - maxSparseTuples(0), sizeSparseTuples(0), cacheUsed(0), + maxSparseTuples(0), sizeSparseTuples(0), maxSparseId(0), cacheUsed(0), cacheSize(0), storedNodes(0), totalNodes(0), nodesCacheHits(0), nodesCacheLookups(0), warn_node_order(0) { diff --git a/node-ram-cache.hpp b/node-ram-cache.hpp index 13bc8d75f..66946cb13 100644 --- a/node-ram-cache.hpp +++ b/node-ram-cache.hpp @@ -132,6 +132,7 @@ struct node_ram_cache : public boost::noncopyable ramNodeID *sparseBlock; int64_t maxSparseTuples; int64_t sizeSparseTuples; + osmid_t maxSparseId; int64_t cacheUsed, cacheSize; osmid_t storedNodes, totalNodes; From af8a35c1ef0822fbdd61a958c9a69235b69e6b85 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 16 Jun 2015 22:14:19 +0200 Subject: [PATCH 4/6] restrict number of blocks in ram cache to 32bit Further reduces ram cache on 64bit machines. --- node-ram-cache.cpp | 22 +++++++++++++--------- node-ram-cache.hpp | 4 ++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/node-ram-cache.cpp b/node-ram-cache.cpp index b349bf21f..23f8d248c 100644 --- a/node-ram-cache.cpp +++ b/node-ram-cache.cpp @@ -60,7 +60,7 @@ int ramNode::scale; -static int id2block(osmid_t id) +static int32_t id2block(osmid_t id) { /* + NUM_BLOCKS/2 allows for negative IDs */ return (id >> BLOCK_SHIFT) + NUM_BLOCKS/2; @@ -71,7 +71,7 @@ static int id2offset(osmid_t id) return id & (PER_BLOCK-1); } -static osmid_t block2id(int block, int offset) +static osmid_t block2id(int32_t block, int offset) { return (((osmid_t) block - NUM_BLOCKS/2) << BLOCK_SHIFT) + (osmid_t) offset; } @@ -131,9 +131,8 @@ int node_ram_cache::set_sparse(osmid_t id, const ramNode &coord) { } int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { - int block = id2block(id); - int offset = id2offset(id); - int i = 0; + int32_t const block = id2block(id); + int const offset = id2offset(id); if (maxBlocks == 0) return 1; @@ -157,7 +156,7 @@ int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { blocks[block].nodes = next_chunk(); } else { /* previous block was not dense enough, so push it into the sparse node cache instead */ - for (i = 0; i < (1 << BLOCK_SHIFT); i++) { + for (int i = 0; i < (1 << BLOCK_SHIFT); i++) { if (queue[usedBlocks -1]->nodes[i].is_valid()) { set_sparse(block2id(queue[usedBlocks - 1]->block_offset, i), queue[usedBlocks -1]->nodes[i]); @@ -199,7 +198,7 @@ int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { * current head of the tree down to the right level to restore the * priority queue invariant. Upto log(maxBlocks) iterations */ - i=0; + int i = 0; while( 2*i+1 < usedBlocks - 1 ) { if( queue[2*i+1]->used() <= queue[2*i+2]->used() ) { if( queue[i]->used() > queue[2*i+1]->used() ) { @@ -280,8 +279,8 @@ int node_ram_cache::get_sparse(osmNode *out, osmid_t id) { } int node_ram_cache::get_dense(osmNode *out, osmid_t id) { - int block = id2block(id); - int offset = id2offset(id); + int32_t const block = id2block(id); + int const offset = id2offset(id); if (!blocks[block].nodes) return 1; @@ -395,6 +394,11 @@ node_ram_cache::~node_ram_cache() { } int node_ram_cache::set(osmid_t id, double lat, double lon, const taglist_t &) { + if ((id > 0 && id >> BLOCK_SHIFT >> 32) || (id < 0 && ~id >> BLOCK_SHIFT >> 32 )) { + fprintf(stderr, "\nAbsolute node IDs must not be larger than %lld (got %lld)\n", + 1ULL << 42, (long long) id); + util::exit_nicely(); + } totalNodes++; /* if ALLOC_DENSE and ALLOC_SPARSE are set, send it through * ram_nodes_set_dense. If a block is non dense, it will automatically diff --git a/node-ram-cache.hpp b/node-ram-cache.hpp index 66946cb13..c292c2a8a 100644 --- a/node-ram-cache.hpp +++ b/node-ram-cache.hpp @@ -97,9 +97,9 @@ class ramNodeBlock { int used() const { return _used >> 1; } ramNode *nodes; - int block_offset; + int32_t block_offset; private: - int _used; // 0-bit indicates dirty + int32_t _used; // 0-bit indicates dirty }; struct node_ram_cache : public boost::noncopyable From 6cf901e609f5fb1faf79c0ca5b79f2dc5264cee9 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 22 Jun 2015 22:53:24 +0200 Subject: [PATCH 5/6] fix int overflows from reduced block_offset --- node-persistent-cache.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp index 43243d9ba..f10433e5f 100644 --- a/node-persistent-cache.cpp +++ b/node-persistent-cache.cpp @@ -52,7 +52,7 @@ void node_persistent_cache::writeout_dirty_nodes() if (readNodeBlockCache[i].dirty()) { if (lseek64(node_cache_fd, - (readNodeBlockCache[i].block_offset + ((osmid_t) readNodeBlockCache[i].block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(ramNode) + sizeof(persistentCacheHeader), @@ -217,7 +217,7 @@ int node_persistent_cache::load_block(osmid_t block_offset) if (readNodeBlockCache[block_id].dirty()) { if (lseek64(node_cache_fd, - (readNodeBlockCache[block_id].block_offset + ((osmid_t) readNodeBlockCache[block_id].block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(ramNode) + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) { fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", @@ -236,7 +236,7 @@ int node_persistent_cache::load_block(osmid_t block_offset) } if (readNodeBlockCache[block_id].nodes) { - remove_from_cache_idx(readNodeBlockCache[block_id].block_offset); + remove_from_cache_idx((osmid_t) readNodeBlockCache[block_id].block_offset); new(readNodeBlockCache[block_id].nodes) ramNode[READ_NODE_BLOCK_SIZE]; } else { readNodeBlockCache[block_id].nodes = new ramNode[READ_NODE_BLOCK_SIZE]; @@ -297,7 +297,7 @@ void node_persistent_cache::nodes_set_create_writeout_block() * node cache file in buffer cache therefore duplicates the data wasting 16GB of ram. * Therefore tell the OS not to cache the node-persistent-cache during initial import. * */ - if (sync_file_range(node_cache_fd, writeNodeBlock.block_offset*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + + if (sync_file_range(node_cache_fd, (osmid_t) writeNodeBlock.block_offset*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode), SYNC_FILE_RANGE_WRITE) < 0) { fprintf(stderr, "Info: Sync_file_range writeout has an issue. This shouldn't be anything to worry about.: %s\n", @@ -305,7 +305,7 @@ void node_persistent_cache::nodes_set_create_writeout_block() }; if (writeNodeBlock.block_offset > 16) { - if(sync_file_range(node_cache_fd, (writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + + if(sync_file_range(node_cache_fd, ((osmid_t) writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode), SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER) < 0) { fprintf(stderr, "Info: Sync_file_range block has an issue. This shouldn't be anything to worry about.: %s\n", @@ -313,7 +313,7 @@ void node_persistent_cache::nodes_set_create_writeout_block() } #ifdef HAVE_POSIX_FADVISE - if (posix_fadvise(node_cache_fd, (writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + + if (posix_fadvise(node_cache_fd, ((osmid_t) writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode), POSIX_FADV_DONTNEED) !=0 ) { fprintf(stderr, "Info: Posix_fadvise failed. This shouldn't be anything to worry about.: %s\n", strerror(errno)); @@ -328,7 +328,7 @@ int node_persistent_cache::set_create(osmid_t id, double lat, double lon) assert(!append_mode); assert(!read_mode); - int block_offset = id >> WRITE_NODE_BLOCK_SHIFT; + int32_t block_offset = id >> WRITE_NODE_BLOCK_SHIFT; if (writeNodeBlock.block_offset != block_offset) { @@ -337,7 +337,7 @@ int node_persistent_cache::set_create(osmid_t id, double lat, double lon) nodes_set_create_writeout_block(); /* After writing out the node block, the file pointer is at the next block level */ writeNodeBlock.block_offset++; - cacheHeader.max_initialised_id = (writeNodeBlock.block_offset + cacheHeader.max_initialised_id = ((osmid_t) writeNodeBlock.block_offset << WRITE_NODE_BLOCK_SHIFT) - 1; } if (writeNodeBlock.block_offset > block_offset) @@ -464,7 +464,7 @@ void node_persistent_cache::set_read_mode() nodes_set_create_writeout_block(); writeNodeBlock.reset_used(); writeNodeBlock.block_offset++; - cacheHeader.max_initialised_id = (writeNodeBlock.block_offset + cacheHeader.max_initialised_id = ((osmid_t) writeNodeBlock.block_offset << WRITE_NODE_BLOCK_SHIFT) - 1; /* write out the header */ From b97aaa4a3a3bd783eae6e49d5cf90f060237942f Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 23 Jun 2015 19:53:04 +0200 Subject: [PATCH 6/6] remove debug output --- node-persistent-cache.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp index f10433e5f..3689ce7f7 100644 --- a/node-persistent-cache.cpp +++ b/node-persistent-cache.cpp @@ -460,7 +460,6 @@ void node_persistent_cache::set_read_mode() if (writeNodeBlock.dirty()) { assert(!append_mode); - fprintf(stderr, "Switching to read mode\n"); nodes_set_create_writeout_block(); writeNodeBlock.reset_used(); writeNodeBlock.block_offset++; @@ -482,8 +481,6 @@ void node_persistent_cache::set_read_mode() } read_mode = true; - - fprintf(stderr, "Switching to read mode done\n"); } node_persistent_cache::node_persistent_cache(const options_t *options, int append,