Skip to content

Commit

Permalink
refs #6449 Extracted NexusIO from box controller
Browse files Browse the repository at this point in the history 
and enabled number of other small things related to file-based boxes
  • Loading branch information
@abuts @martyngigg
abuts authored and martyngigg committed Apr 10, 2013
1 parent 8915902 commit ff87e39
Show file tree
Hide file tree
Showing 14 changed files with 752 additions and 566 deletions.
1 change: 1 addition & 0 deletions Code/Mantid/Framework/API/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ set ( INC_FILES
inc/MantidAPI/IAlgorithm.h
inc/MantidAPI/IArchiveSearch.h
inc/MantidAPI/IBackgroundFunction.h
inc/MantidAPI/IBoxControllerIO.h
inc/MantidAPI/ICatalog.h
inc/MantidAPI/IConstraint.h
inc/MantidAPI/ICostFunction.h
Expand Down
80 changes: 16 additions & 64 deletions Code/Mantid/Framework/API/inc/MantidAPI/BoxController.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
#include "MantidKernel/MultiThreaded.h"
#include "MantidKernel/System.h"
#include "MantidKernel/ThreadPool.h"
#include "MantidKernel/Exception.h"
#include "MantidAPI/IBoxControllerIO.h"
#include <nexus/NeXusFile.hpp>
#include <boost/shared_ptr.hpp>
#include <vector>
Expand Down Expand Up @@ -34,21 +36,20 @@ namespace API
* @return BoxController instance
*/
BoxController(size_t nd)
:nd(nd), m_maxId(0),m_SplitThreshold(1024), m_numSplit(1), m_file(NULL), m_diskBuffer()//, m_useWriteBuffer(true)
:nd(nd), m_maxId(0),m_SplitThreshold(1024), m_numSplit(1), m_diskBuffer()//, m_useWriteBuffer(true)
{
// TODO: Smarter ways to determine all of these values
m_maxDepth = 5;
m_addingEvents_eventsPerTask = 1000;
m_addingEvents_numTasksPerBlock = Kernel::ThreadPool::getNumPhysicalCores() * 5;
m_splitInto.resize(this->nd, 1);
m_DataChunk = 10000;
resetNumBoxes();
}

BoxController(const BoxController & other );

virtual ~BoxController();

// create new box controller from the existing one
boost::shared_ptr<BoxController> clone()const;
/// Serialize
std::string toXMLString() const;

Expand Down Expand Up @@ -264,19 +265,6 @@ namespace API
// Return true if the average # of events per box is big enough to split.
return ((eventsAdded / numMDBoxes) > m_SplitThreshold);
}
/**The method returns the data chunk (continious part of the NeXus array) used to write data on HDD */
size_t getDataChunk()const
{
return m_DataChunk;
}
/** The method used to load nexus data chunk size to the box controller. Used when loading MDEvent nexus file
Disabled at the moment as it is unclear how to get acsess to physical size of NexUs data set and optimal chunk size should be physical Nexus chunk size
*/
//void setChunkSize(size_t chunkSize)
//{
// m_DataChunk = chunkSize;
// }

//-----------------------------------------------------------------------------------
/** Call to track the number of MDBoxes are contained in the MDEventWorkspace
* This should be called when a MDBox gets split into a MDGridBox.
Expand Down Expand Up @@ -359,33 +347,6 @@ namespace API
m_mutexNumMDBoxes.unlock();
}


//-----------------------------------------------------------------------------------
/** @return the open NeXus file handle. NULL if not file-backed. */
::NeXus::File * getFile() const
{ return m_file; }

/** Sets the open Nexus file to use with file-based back-end
* @param file :: file handle
* @param filename :: full path to the file
* @param fileLength :: length of the file being open, in number of events in this case */
void setFile(::NeXus::File * file, const std::string & filename, const uint64_t fileLength)
{
m_file = file;
m_filename = filename;
m_diskBuffer.setFileLength(fileLength);
}

/** @return true if the MDEventWorkspace is backed by a file */
bool isFileBacked() const
{ return m_file != NULL; }

/// @return the full path to the file open as the file-based back end.
const std::string & getFilename() const
{ return m_filename; }

void closeFile(bool deleteFile = false);

//-----------------------------------------------------------------------------------
/** Return the DiskBuffer for disk caching */
const Mantid::Kernel::DiskBuffer & getDiskBuffer() const
Expand All @@ -398,6 +359,14 @@ namespace API
/** Return true if the DiskBuffer should be used -- in current edition it is always used*/
bool useWriteBuffer() const{return true;}
// { return m_useWriteBuffer; }
/// Returns if current box controller is file backed. Assumes that BC(workspace) is fileBackd if fileIO is defined;
bool isFileBacked()const
{return m_fileIO;}
/// returns the pointer to the class, responsible for fileIO operations;
IBoxControllerIO * getFileIO()
{return m_fileIO.get();}
/// makes box controller file based by providing class, responsible for fileIO.
void setFileBacked(IBoxControllerIO *newFileIO,const std::string &fileName="");

//-----------------------------------------------------------------------------------
/** Set the memory-caching parameters for a file-backed
Expand Down Expand Up @@ -451,6 +420,8 @@ namespace API
}

private:
/// box controller is an ws-based singleton so it should not be possible to copy it
BoxController(const BoxController & other );
/// Number of dimensions
size_t nd;

Expand Down Expand Up @@ -498,12 +469,7 @@ namespace API
/// Mutex for getting IDs
Mantid::Kernel::Mutex m_idMutex;

/// Filename of the file backend
std::string m_filename;

/// Open file handle to the file back-end
::NeXus::File * m_file;

boost::shared_ptr<IBoxControllerIO> m_fileIO;
/// Instance of the disk-caching MRU list.
mutable Mantid::Kernel::DiskBuffer m_diskBuffer;

Expand All @@ -512,21 +478,7 @@ namespace API

/// Number of bytes in a single MDLeanEvent<> of the workspace.
size_t m_bytesPerEvent;
/// The size of the events block which can be written in the neXus array at once (continious part of the data block)
size_t m_DataChunk;
public:

static void prepareEventNexusData(::NeXus::File * file,const size_t DataChunk,const size_t nColumns,const std::string &descr);

//---------------------------------------------------------------------------------------------
/** Open the NXS event data blocks for loading.
*
* @param file :: open NXS file.
* @return the number of events currently in the data field.
*/
static uint64_t openEventNexusData(::NeXus::File * file);

static void closeNexusData(::NeXus::File * file);
};


Expand Down
52 changes: 52 additions & 0 deletions Code/Mantid/Framework/API/inc/MantidAPI/IBoxControllerIO.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#ifndef H_IBOXCONTROLLER_IO
#define H_IBOXCONTROLLER_IO
#include "MantidKernel/System.h"

namespace Mantid
{
namespace API
{

/** The header describes interface to IO Operations perfomed by the box controller
* May be replaced by a boos filestream in a future.
* @date March 21, 2013
Copyright &copy; 2007-2013 ISIS Rutherford Appleton Laboratory & NScD Oak Ridge National Laboratory
This file is part of Mantid.
Mantid is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
Mantid is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
File change history is stored at: <https://github.com/mantidproject/mantid>.
Code Documentation is available at: <http://doxygen.mantidproject.org>
*/

class DLLExport IBoxControllerIO
{
public:
virtual bool openFile(const std::string &fileName)=0;
virtual bool isOpened()const=0;
virtual const std::string &getFileName()const=0;

virtual void saveBlock(void const * const /* Block */, const uint64_t /*blockPosition*/,const size_t /*blockSize*/)=0;
virtual void loadBlock(void * const /* Block */, const uint64_t /*blockPosition*/,const size_t /*blockSize*/)=0;
virtual void flushData()=0;
virtual void closeFile()=0;

virtual ~IBoxControllerIO(){}
};
}
}
#endif
2 changes: 1 addition & 1 deletion Code/Mantid/Framework/API/inc/MantidAPI/IMDNode.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ namespace API
class IMDNode
{
public:
virtual ~IMDNode(){};
virtual ~IMDNode(){};
//---------------- ISAVABLE
virtual Kernel::ISaveable *const getISaveable()=0;
virtual Kernel::ISaveable *const getISaveable()const=0;
Expand Down
101 changes: 34 additions & 67 deletions Code/Mantid/Framework/API/src/BoxController.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,38 @@ namespace API
{

//-----------------------------------------------------------------------------------
/** Copy constructor
/** create new box controller from the existing one
* @param pointer to new instance of a class responsible for boxes IO-operations
*/
BoxController_sptr BoxController::clone()const
{
BoxController * theClone = new BoxController(*this);
// reset the clone file IO controller to avoid dublicated file based operations for different box controllers
theClone->m_fileIO.reset();
return BoxController_sptr(theClone);
}
/** makes box controller file based by providing class, responsible for fileIO. The box controller become responsible for the FileIO pointer
*@param newFileIO -- instance of the box controller responsible for the IO;
*@param fileName -- if newFileIO comes without opened file, this is the file name to open for the file based IO operations
*/
void BoxController::setFileBacked(IBoxControllerIO *newFileIO,const std::string &fileName)
{
if(!newFileIO->isOpened())
newFileIO->openFile(fileName);

if(!newFileIO->isOpened())
{
delete newFileIO;
throw(Kernel::Exception::FileError("Can not open target file for filebased box controller ",fileName));
}

if(this->m_fileIO) // should happen in destructor anyway but just to be carefull about it
this->m_fileIO->closeFile();

this->m_fileIO = boost::shared_ptr<IBoxControllerIO>(newFileIO);
}

/*Private Copy constructor used in cloning */
BoxController::BoxController(const BoxController & other)
: nd(other.nd), m_maxId(other.m_maxId),
m_SplitThreshold(other.m_SplitThreshold),
Expand All @@ -36,17 +66,16 @@ namespace API
m_addingEvents_numTasksPerBlock(other.m_addingEvents_numTasksPerBlock),
m_numMDBoxes(other.m_numMDBoxes),
m_numMDGridBoxes(other.m_numMDGridBoxes),
m_maxNumMDBoxes(other.m_maxNumMDBoxes),
m_filename(other.m_filename),
m_file(other.m_file),
m_maxNumMDBoxes(other.m_maxNumMDBoxes),
m_bytesPerEvent(other.m_bytesPerEvent)
{
}

/// Destructor
BoxController::~BoxController()
{
this->closeFile();
if(m_fileIO)
m_fileIO->closeFile();
}
/**reserve range of id-s for use on set of adjacent boxes.
* Needed to be thread safe as adjacent boxes have to have subsequent ID-s
Expand Down Expand Up @@ -155,68 +184,6 @@ namespace API
}


//------------------------------------------------------------------------------------------------------
/** Close the open file for the back-end, if any
* Note: this does not save any data that might be, e.g., in the MRU.
* @param deleteFile :: if true, will delete the file. Default false.
*/
void BoxController::closeFile(bool deleteFile)
{
if (m_file)
{
m_file->close();
m_file = NULL;
}
if (deleteFile && !m_filename.empty())
{
Poco::File file(m_filename);
if (file.exists()) file.remove();
m_filename = "";
}
}


void BoxController::prepareEventNexusData(::NeXus::File * file, const size_t chunkSize,const size_t nColumns,const std::string &descr)
{
std::vector<int> dims(2,0);
dims[0] = NX_UNLIMITED;
// One point per dimension, plus signal, plus error, plus runIndex, plus detectorID = nd+4
dims[1] = int(nColumns);

// Now the chunk size.
std::vector<int> chunk(dims);
chunk[0] = int(chunkSize);

// Make and open the data
#ifdef COORDT_IS_FLOAT
file->makeCompData("event_data", ::NeXus::FLOAT32, dims, ::NeXus::NONE, chunk, true);
#else
file->makeCompData("event_data", ::NeXus::FLOAT64, dims, ::NeXus::NONE, chunk, true);
#endif

// A little bit of description for humans to read later
file->putAttr("description", descr);

}

//---------------------------------------------------------------------------------------------
/** Open the NXS data blocks for loading.
* The data should have been created before.
*
* @param file :: open NXS file.
* @return the number of events currently in the data field.
*/
uint64_t BoxController::openEventNexusData(::NeXus::File * file)
{
// Open the data
file->openData("event_data");
// Return the size of dimension 0 = the number of events in the field
return uint64_t(file->getInfo().dims[0]);
}
void BoxController::closeNexusData(::NeXus::File * file)
{
file->closeData();
}


} // namespace Mantid
Expand Down

0 comments on commit ff87e39

Please sign in to comment.