Skip to content

Commit

Permalink
merge with 'master' and improvements in ChunkedArrayHDF5
Browse files Browse the repository at this point in the history
  • Loading branch information
@ukoethe
ukoethe committed Apr 9, 2014
2 parents f4d5f0f + c7f9617 commit 4fe8f6d
Show file tree
Hide file tree
Showing 7 changed files with 145 additions and 39 deletions.
14 changes: 14 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,20 @@ IF (CMAKE_COMPILER_IS_GNUCXX)
ENDIF()
ENDIF ()

IF("${CMAKE_CXX_COMPILER_ID}" MATCHES "(GNU|Clang)")
set(SUFFICIENT_TEMPLATE_DEPTH FALSE)
TRY_COMPILE(SUFFICIENT_TEMPLATE_DEPTH
${CMAKE_BINARY_DIR} ${PROJECT_SOURCE_DIR}/config/checkTemplateDepth.cxx
COMPILE_DEFINITIONS "-DDEPTH=900")

IF(SUFFICIENT_TEMPLATE_DEPTH)
MESSAGE(STATUS "Checking template recursion depth: ok")
ELSE()
SET(CMAKE_CXX_FLAGS "-ftemplate-depth=900 ${CMAKE_CXX_FLAGS}" CACHE STRING "Flags used by the compiler during all build types" FORCE)
MESSAGE(STATUS "Checking template recursion depth: using -ftemplate-depth=900")
ENDIF()
ENDIF()

# if(CMAKE_COMPILER_IS_GNUCXX)
# exec_program(
# ${CMAKE_CXX_COMPILER}
Expand Down
16 changes: 16 additions & 0 deletions config/checkTemplateDepth.cxx
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
template <int N>
struct Sum
{
static const int sum = 1 + Sum<N-1>::sum;
};

template <>
struct Sum<0>
{
static const int sum = 0;
};

int main()
{
static const int sum = Sum<DEPTH>::sum;
}
7 changes: 5 additions & 2 deletions include/vigra/hdf5impex.hxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1089,6 +1089,11 @@ class HDF5File
return fileHandle_.use_count();
}

bool isOpen() const
{
return fileHandle_ != 0;
}

bool isReadOnly() const
{
return read_only_;
Expand All @@ -1113,8 +1118,6 @@ class HDF5File
}

/** \brief Close the current file.
Calls close() on the present file and then assigns itself to the same file and group as \a other.
*/
void close()
{
Expand Down
9 changes: 8 additions & 1 deletion include/vigra/multi_array_chunked.hxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1771,8 +1771,10 @@ class ChunkedArray
}

// NOTE: this function must only be called while we hold the chunk_lock_
void cleanCache(int how_many)
void cleanCache(int how_many = -1)
{
if(how_many == -1)
how_many = cache_.size();
for(; cache_.size() > cacheMaxSize() && how_many > 0; --how_many)
{
Handle * handle = cache_.front();
Expand Down Expand Up @@ -2020,6 +2022,11 @@ class ChunkedArray
void setCacheMaxSize(std::size_t c)
{
cache_max_size_ = c;
if(c < cache_.size())
{
threading::lock_guard<threading::mutex> guard(*chunk_lock_);
cleanCache();
}
}

iterator begin()
Expand Down
58 changes: 47 additions & 11 deletions include/vigra/multi_array_chunked_hdf5.hxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,29 +265,56 @@ class ChunkedArrayHDF5

~ChunkedArrayHDF5()
{
typename ChunkStorage::iterator i = this->handle_array_.begin(),
end = this->handle_array_.end();
for(; i != end; ++i)
{
if(i->pointer_)
delete static_cast<Chunk*>(i->pointer_);
i->pointer_ = 0;
}
closeImpl(true);
}

void close()
{
closeImpl(false);
}

void closeImpl(bool force_destroy)
{
flushToDiskImpl(true, force_destroy);
file_.close();
}

void flushToDisk()
{
flushToDiskImpl(false, false);
}

void flushToDiskImpl(bool destroy, bool force_destroy)
{
if(file_.isReadOnly())
return;

threading::lock_guard<threading::mutex> guard(*this->chunk_lock_);
typename ChunkStorage::iterator i = this->handle_array_.begin(),
end = this->handle_array_.end();
if(destroy && !force_destroy)
{
for(; i != end; ++i)
{
vigra_precondition(i->chunk_state_ <= 0,
"ChunkedArrayHDF5::close(): cannot close file because there are active chunks.");
}
i = this->handle_array_.begin();
}
for(; i != end; ++i)
{
Chunk * chunk = static_cast<Chunk*>(i->pointer_);
if(chunk)
if(!chunk)
continue;
if(destroy)
{
delete chunk;
i->pointer_ = 0;
}
else
{
chunk->write(false);
}
}
file_.flushToDisk();
}
Expand All @@ -299,6 +326,8 @@ class ChunkedArrayHDF5

virtual pointer loadChunk(ChunkBase<N, T> ** p, shape_type const & index)
{
vigra_precondition(file_.isOpen(),
"ChunkedArrayHDF5::loadChunk(): file was already closed.");
if(*p == 0)
{
*p = new Chunk(this->chunkShape(index), index*this->chunk_shape_, this, alloc_);
Expand All @@ -309,8 +338,10 @@ class ChunkedArrayHDF5

virtual bool unloadChunk(ChunkBase<N, T> * chunk, bool /* destroy */)
{
if(!file_.isOpen())
return true;
static_cast<Chunk *>(chunk)->write();
return false; // never destroys the data
return false;
}

virtual std::string backend() const
Expand All @@ -330,7 +361,12 @@ class ChunkedArrayHDF5
return sizeof(Chunk) + sizeof(SharedChunkHandle<N, T>);
}

std::string const & datasetName() const
std::string fileName() const
{
return file_.filename();
}

std::string datasetName() const
{
return dataset_name_;
}
Expand Down
7 changes: 7 additions & 0 deletions vigranumpy/lib/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,13 @@ def __getattr__(self, name):
standardArrayType = arraytypes.VigraArray
defaultAxistags = arraytypes.VigraArray.defaultAxistags

from vigranumpycore import ChunkedArrayFull, ChunkedArrayLazy, ChunkedArrayCompressed, ChunkedArrayTmpFile, Compression
try:
from vigranumpycore import ChunkedArrayHDF5, HDF5Mode
except:
pass


from impex import readImage, readVolume

def readHDF5(filenameOrGroup, pathInFile, order=None):
Expand Down
73 changes: 48 additions & 25 deletions vigranumpy/src/core/multi_array_chunked.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,19 +241,27 @@ int numpyScalarTypeNumber(python::object obj)
return typeNum;
}

template <unsigned int N, class T>
template <class Array>
PyObject *
ptr_to_python(ChunkedArray<N, T> * a, std::string const & axistags)
ptr_to_python(Array * a, python::object axistags)
{
PyObject * pya = python::to_python_indirect<ChunkedArray<N, T>*,
PyObject * pya = python::to_python_indirect<Array*,
python::detail::make_owning_holder>()(a);
AxisTags at(axistags);
vigra_precondition(at.size() == 0 || at.size() == N,
"ChunkedArray(): axistags have invalid length.");
if(at.size() == N)
if(axistags != python::object())
{
int res = PyObject_SetAttrString(pya, "axistags", python::object(at).ptr());
pythonToCppException(res != 0);
AxisTags at;
if(PyString_Check(axistags.ptr()))
at = AxisTags(python::extract<std::string>(axistags)());
else
at = AxisTags(python::extract<AxisTags const &>(axistags)());
int N = Array::shape_type::static_size;
vigra_precondition(at.size() == 0 || at.size() == N,
"ChunkedArray(): axistags have invalid length.");
if(at.size() == N)
{
int res = PyObject_SetAttrString(pya, "axistags", python::object(at).ptr());
pythonToCppException(res != 0);
}
}
return pya;
}
Expand All @@ -271,7 +279,7 @@ template <unsigned int N>
PyObject *
construct_ChunkedArrayFull(TinyVector<MultiArrayIndex, N> const & shape,
python::object dtype, double fill_value,
std::string const & axistags)
python::object axistags)
{
switch(numpyScalarTypeNumber(dtype))
{
Expand Down Expand Up @@ -303,7 +311,7 @@ construct_ChunkedArrayLazy(TinyVector<MultiArrayIndex, N> const & shape,
python::object dtype,
TinyVector<MultiArrayIndex, N> const & chunk_shape,
double fill_value,
std::string const & axistags)
python::object axistags)
{
switch(numpyScalarTypeNumber(dtype))
{
Expand Down Expand Up @@ -339,7 +347,7 @@ construct_ChunkedArrayCompressed(TinyVector<MultiArrayIndex, N> const & shape,
TinyVector<MultiArrayIndex, N> const & chunk_shape,
int cache_max,
double fill_value,
std::string const & axistags)
python::object axistags)
{
switch(numpyScalarTypeNumber(dtype))
{
Expand Down Expand Up @@ -378,7 +386,7 @@ construct_ChunkedArrayTmpFile(TinyVector<MultiArrayIndex, N> const & shape,
int cache_max,
std::string path,
double fill_value,
std::string const & axistags)
python::object axistags)
{
switch(numpyScalarTypeNumber(dtype))
{
Expand All @@ -400,7 +408,7 @@ construct_ChunkedArrayTmpFile(TinyVector<MultiArrayIndex, N> const & shape,
#ifdef HasHDF5

template <class T, int N>
ChunkedArray<N, T> *
ChunkedArrayHDF5<N, T> *
construct_ChunkedArrayHDF5Impl(HDF5File const & file,
std::string datasetName,
TinyVector<MultiArrayIndex, N> const & shape,
Expand All @@ -426,7 +434,7 @@ construct_ChunkedArrayHDF5Impl(HDF5File const & file,
TinyVector<MultiArrayIndex, N> const & chunk_shape,
int cache_max,
double fill_value,
std::string const & axistags)
python::object axistags)
{
int dtype_code = NPY_FLOAT32;

Expand Down Expand Up @@ -469,7 +477,7 @@ construct_ChunkedArrayHDF5Impl(HDF5File const & file,
python::object py_chunk_shape,
int cache_max,
double fill_value,
std::string const & axistags)
python::object axistags)
{
int ndim = 0;
bool has_shape = PySequence_Check(py_shape.ptr());
Expand Down Expand Up @@ -565,7 +573,7 @@ construct_ChunkedArrayHDF5(std::string filename,
python::object chunk_shape,
int cache_max,
double fill_value,
std::string const & axistags)
python::object axistags)
{
bool file_exists = isHDF5(filename.c_str());
if(mode == HDF5File::Default)
Expand Down Expand Up @@ -601,7 +609,7 @@ construct_ChunkedArrayHDF5id(hid_t file_id,
python::object chunk_shape,
int cache_max,
double fill_value,
std::string const & axistags)
python::object axistags)
{
HDF5HandleShared handle(file_id, 0, "");
HDF5File file(handle);
Expand Down Expand Up @@ -669,6 +677,19 @@ void defineChunkedArrayImpl()
.def("__setitem__", &ChunkedArray_setitem2<N, T>)
;

#ifdef HasHDF5
typedef ChunkedArrayHDF5<N, T> ArrayHDF5;
class_<ChunkedArrayHDF5<N, T>, bases<Array>, boost::noncopyable>("ChunkedArrayHDF5", no_init)
.def("close", &ArrayHDF5::close)
.def("flush", &ArrayHDF5::flushToDisk)
.add_property("filename", &ArrayHDF5::fileName,
"\nname of the file backend of this array.\n")
.add_property("dataset_name", &ArrayHDF5::datasetName,
"\nname of the dataset backend of this array.\n")
.add_property("readonly", &ArrayHDF5::isReadOnly,
"\nTrue if this array is read-only.\n")
;
#endif
}

template <unsigned int N>
Expand All @@ -680,16 +701,16 @@ void defineChunkedArrayFactories()
docstring_options doc_options(true, false, false);

def("ChunkedArrayFull", &construct_ChunkedArrayFull<N>,
(arg("shape"), arg("dtype")=defaultDtype(), arg("fill_value")=0.0, arg("axistags")=""));
(arg("shape"), arg("dtype")=defaultDtype(), arg("fill_value")=0.0, arg("axistags")=python::object()));
def("ChunkedArrayLazy", &construct_ChunkedArrayLazy<N>,
(arg("shape"), arg("dtype")=defaultDtype(),
arg("chunk_shape")=shape_type(), arg("fill_value")=0.0, arg("axistags")=""));
arg("chunk_shape")=shape_type(), arg("fill_value")=0.0, arg("axistags")=python::object()));
def("ChunkedArrayCompressed", &construct_ChunkedArrayCompressed<N>,
(arg("shape"), arg("method")=LZ4, arg("dtype")=defaultDtype(), arg("chunk_shape")=shape_type(),
arg("cache_max")=-1, arg("fill_value")=0.0, arg("axistags")=""));
(arg("shape"), arg("compression")=LZ4, arg("dtype")=defaultDtype(), arg("chunk_shape")=shape_type(),
arg("cache_max")=-1, arg("fill_value")=0.0, arg("axistags")=python::object()));
def("ChunkedArrayTmpFile", &construct_ChunkedArrayTmpFile<N>,
(arg("shape"), arg("dtype")=defaultDtype(), arg("chunk_shape")=shape_type(),
arg("cache_max")=-1, arg("path")="", arg("fill_value")=0.0, arg("axistags")=""));
arg("cache_max")=-1, arg("path")="", arg("fill_value")=0.0, arg("axistags")=python::object()));
}

void defineChunkedArray()
Expand Down Expand Up @@ -754,11 +775,13 @@ void defineChunkedArray()
def("ChunkedArrayHDF5", &construct_ChunkedArrayHDF5id,
(arg("file_id"), arg("dataset_name"), arg("shape")=python::object(),
arg("dtype")=python::object(), arg("mode")=HDF5File::ReadOnly, arg("compression")=ZLIB_FAST,
arg("chunk_shape")=python::object(), arg("cache_max")=-1, arg("fill_value")=0.0, arg("axistags")=""));
arg("chunk_shape")=python::object(), arg("cache_max")=-1, arg("fill_value")=0.0,
arg("axistags")=python::object()));
def("ChunkedArrayHDF5", &construct_ChunkedArrayHDF5,
(arg("file_name"), arg("dataset_name"), arg("shape")=python::object(),
arg("dtype")=python::object(), arg("mode")=HDF5File::Default, arg("compression")=ZLIB_FAST,
arg("chunk_shape")=python::object(), arg("cache_max")=-1, arg("fill_value")=0.0, arg("axistags")=""));
arg("chunk_shape")=python::object(), arg("cache_max")=-1, arg("fill_value")=0.0,
arg("axistags")=python::object()));
#endif
}

Expand Down

0 comments on commit 4fe8f6d

Please sign in to comment.