Skip to content

Commit

Permalink
Use copyPointsFast for situations where the schema
Browse files Browse the repository at this point in the history 
size of the cache's blocks are the same size as the users's buffer.

Don't create a new vector of block pointers every time we do a
lookup. Instead let the user ask for a reference to the vector of
pointers that lives on the filters.cache.
  • Loading branch information
@hobu
hobu committed Mar 21, 2013
1 parent 430bac5 commit 273cdd5
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 20 deletions.
4 changes: 3 additions & 1 deletion include/pdal/filters/Cache.hpp
View file
Expand Up @@ -74,7 +74,7 @@ class PDAL_DLL Cache : public Filter
// this is const only because the m_cache itself is mutable
void addToCache(boost::uint64_t pointIndex, const PointBuffer& data) const;

std::vector<PointBuffer const*> lookup(boost::uint64_t pointPosition, boost::uint32_t count) const;
void lookup(boost::uint64_t pointPosition, boost::uint32_t count) const;
bool isCached(boost::uint64_t pointPosition, boost::uint32_t count) const;
// clear cache (but leave cache params unchanged)
void resetCache();
Expand Down Expand Up @@ -116,6 +116,7 @@ class PDAL_DLL Cache : public Filter
boost::uint32_t calculateNumberOfBlocks(boost::uint32_t CacheBlockSize,
boost::uint64_t totalNumberOfPoints) const;

inline std::vector<PointBuffer const*> const& getCachedBlocks() const { return m_blocks; }

pdal::StageSequentialIterator* createSequentialIterator(PointBuffer& buffer) const;
pdal::StageRandomIterator* createRandomIterator(PointBuffer& buffer) const;
Expand All @@ -129,6 +130,7 @@ class PDAL_DLL Cache : public Filter
// these is mutable to allow const-ness for updating stats
// BUG: need to make thread-safe
mutable PointBufferCache* m_cache;
mutable std::vector<PointBuffer const*> m_blocks;

boost::uint32_t m_maxCacheBlocks;
boost::uint32_t m_cacheBlockSize;
Expand Down
62 changes: 43 additions & 19 deletions src/filters/Cache.cpp
View file
Expand Up @@ -127,11 +127,10 @@ bool Cache::isCached(boost::uint64_t pointPosition, boost::uint32_t count) const
return true;
}

std::vector<PointBuffer const*> Cache::lookup(boost::uint64_t pointPosition, boost::uint32_t count) const
void Cache::lookup(boost::uint64_t pointPosition, boost::uint32_t count) const
{

std::vector<PointBuffer const*> empty;
std::vector<PointBuffer const*> output;

m_blocks.clear();

boost::uint32_t startingBlockNumber = pointPosition / getCacheBlockSize();
boost::uint32_t endingBlockNumber = (pointPosition + count - 1) / getCacheBlockSize();
Expand All @@ -146,8 +145,12 @@ std::vector<PointBuffer const*> Cache::lookup(boost::uint64_t pointPosition, boo
{
PointBuffer const* block = m_cache->lookup(startingBlockNumber);
if (!block)
return empty;
output.push_back(block);
{
m_blocks.clear();
return;
}

m_blocks.push_back(block);
} else
{
for (boost::uint32_t i(startingBlockNumber); i <= endingBlockNumber; ++i)
Expand All @@ -162,22 +165,22 @@ std::vector<PointBuffer const*> Cache::lookup(boost::uint64_t pointPosition, boo
{
if (logOutput)
log()->get(logDEBUG4) << "Did not find block " << i << std::endl;
return empty;
m_blocks.clear();
return;

}

output.push_back(block);
m_blocks.push_back(block);
}

}


if (logOutput)
log()->get(logDEBUG2) << "found " << output.size()
log()->get(logDEBUG2) << "found " << m_blocks.size()
<< " block(s) for position "
<< pointPosition << " with size "
<< count << std::endl;
return output;
}

void Cache::getCacheStats(boost::uint64_t& numCacheLookupMisses,
Expand Down Expand Up @@ -284,10 +287,12 @@ boost::uint32_t IteratorBase::copyCachedBlocks( std::vector<PointBuffer cons

boost::uint32_t numPointsCopied(0);

bool logOutput = m_cache_filter.log()->getLevel() > logDEBUG3;
if (m_mapped_buffer != &data)
{
m_dimension_maps.clear();
m_mapped_buffer = &data;
if (logOutput)
m_cache_filter.log()->get(logDEBUG2) << "Wiping dimension map because mapped buffer != current buffer!" << std::endl;
}
assert (m_mapped_buffer == &data);
Expand All @@ -305,6 +310,7 @@ boost::uint32_t IteratorBase::copyCachedBlocks( std::vector<PointBuffer cons
std::pair<PointBuffer const*, DimensionMap const*> p(b, d);
m_dimension_maps.insert(p);
dim_map = d;
if (logOutput)
m_cache_filter.log()->get(logDEBUG2) << "Inserting dimension map entry for block" << std::endl;
} else
{
Expand All @@ -320,7 +326,6 @@ boost::uint32_t IteratorBase::copyCachedBlocks( std::vector<PointBuffer cons

boost::int64_t blockBufferStartingPosition = currentPosition - blockStartingPosition ;

bool logOutput = m_cache_filter.log()->getLevel() > logDEBUG3;
if (logOutput)
{
m_cache_filter.log()->get(logDEBUG4) << "readEndPosition: " << readEndPosition << std::endl;
Expand All @@ -333,12 +338,24 @@ boost::uint32_t IteratorBase::copyCachedBlocks( std::vector<PointBuffer cons
}

// bool areEqual = b->getSchema() == data.getSchema();

// std::cout << "requested: " << data.getSchema() << std::endl;
// std::cout << "cached: " << b->getSchema() << std::endl;

// This is dirty.
if (data.getSchema().getByteSize() == b->getSchema().getByteSize())
{

data.copyPointsFast(userBufferStartingPosition, blockBufferStartingPosition, *b, blockHowMany);
} else
{
PointBuffer::copyLikeDimensions(*b, data,
*dim_map,
blockBufferStartingPosition,
userBufferStartingPosition,
blockHowMany);
}
// m_cache_filter.log()->get(logDEBUG2) << "schemas equal? " << areEqual << std::endl;
PointBuffer::copyLikeDimensions(*b, data,
*dim_map,
blockBufferStartingPosition,
userBufferStartingPosition,
blockHowMany);

blockNumber++;
numPointsCopied = numPointsCopied + blockHowMany;
Expand Down Expand Up @@ -418,12 +435,13 @@ boost::uint32_t Cache::readBufferImpl(PointBuffer& data)
boost::uint32_t blockNumber = currentPointIndex / cacheBlockSize;

bool logOutput = m_cache_filter.log()->getLevel() > logDEBUG3;


std::vector<PointBuffer const*> blocks = m_cache_filter.lookup(currentPointIndex, data.getCapacity());
m_cache_filter.lookup(currentPointIndex, data.getCapacity());
std::vector<PointBuffer const*> const& blocks = m_cache_filter.getCachedBlocks();

if (blocks.size())
{
if (logOutput)
m_cache_filter.log()->get(logDEBUG3) << "sequential read had cache hit for block "
<< blockNumber << " with at point index "
<< currentPointIndex << std::endl;
Expand Down Expand Up @@ -477,6 +495,9 @@ Cache::Cache(const pdal::filters::Cache& filter, PointBuffer& buffer)
, cache::IteratorBase(filter, buffer)
{

bool logOutput = filter.log()->getLevel() > logDEBUG3;

if (logOutput)
filter.log()->get(logDEBUG) << "create random iterator: creating cache of "
<< filter.getMaxCacheBlocks() << " blocks of capacity "
<< filter.getCacheBlockSize() << std::endl;
Expand Down Expand Up @@ -513,10 +534,13 @@ boost::uint32_t Cache::readBufferImpl(PointBuffer& data)
blockNumber = cacheBlockSize / currentPointIndex;
}

std::vector<PointBuffer const*> blocks = m_cache_filter.lookup(currentPointIndex, data.getCapacity());
bool logOutput = m_cache_filter.log()->getLevel() > logDEBUG3;

m_cache_filter.lookup(currentPointIndex, data.getCapacity());
std::vector<PointBuffer const*> const& blocks = m_cache_filter.getCachedBlocks();
if (blocks.size())
{
if (logOutput)
m_cache_filter.log()->get(logDEBUG3) << "random read had cache hit for block "
<< blockNumber << " with at point index "
<< currentPointIndex << std::endl;
Expand Down

0 comments on commit 273cdd5

Please sign in to comment.