Skip to content

Commit

Permalink
refs #6449 fixed problem with disk buffer allocation
Browse files Browse the repository at this point in the history 
(should be unit tests for that particular case)
  • Loading branch information
@abuts @martyngigg
abuts authored and martyngigg committed Apr 10, 2013
1 parent aefc651 commit 3139d65
Show file tree
Hide file tree
Showing 16 changed files with 286 additions and 271 deletions.
10 changes: 4 additions & 6 deletions Code/Mantid/Framework/API/inc/MantidAPI/BoxController.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@ namespace API
* @return BoxController instance
*/
BoxController(size_t nd)
:nd(nd), m_maxId(0),m_SplitThreshold(1024), m_numSplit(1)
:nd(nd), m_maxId(0),m_SplitThreshold(1024), m_numSplit(1),
m_fileIO(boost::shared_ptr<API::IBoxControllerIO>())
{
// TODO: Smarter ways to determine all of these values
m_maxDepth = 5;
m_addingEvents_eventsPerTask = 1000;
m_addingEvents_numTasksPerBlock = Kernel::ThreadPool::getNumPhysicalCores() * 5;
m_splitInto.resize(this->nd, 1);
resetNumBoxes();
m_fileIO.reset();
}

virtual ~BoxController();
Expand Down Expand Up @@ -449,11 +449,9 @@ namespace API
/// Mutex for getting IDs
Mantid::Kernel::Mutex m_idMutex;

// the class which does actual IO operations
// the class which does actual IO operations, including MRU support list
boost::shared_ptr<IBoxControllerIO> m_fileIO;
/// Instance of the disk-caching MRU list.
// mutable Mantid::Kernel::DiskBuffer m_diskBuffer;



/// Number of bytes in a single MDLeanEvent<> of the workspace.
//size_t m_bytesPerEvent;
Expand Down
18 changes: 6 additions & 12 deletions Code/Mantid/Framework/API/src/BoxController.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,10 @@ namespace API
m_addingEvents_numTasksPerBlock(other.m_addingEvents_numTasksPerBlock),
m_numMDBoxes(other.m_numMDBoxes),
m_numMDGridBoxes(other.m_numMDGridBoxes),
m_maxNumMDBoxes(other.m_maxNumMDBoxes)
m_maxNumMDBoxes(other.m_maxNumMDBoxes),
m_fileIO(boost::shared_ptr<API::IBoxControllerIO>())
{
m_fileIO.reset();

}

/// Destructor
Expand Down Expand Up @@ -181,12 +182,13 @@ namespace API
void BoxController::clearFileBacked()
{
if(m_fileIO)
{
{
// flush DB cache
m_fileIO->flushCache();
// close underlying file
m_fileIO->closeFile();
// decrease the sp counter by one and nullify this instance of sp.
m_fileIO.reset();
m_fileIO.reset();// = boost::shared_ptr<API::IBoxControllerIO>();
}
}
/** makes box controller file based by providing class, responsible for fileIO. The box controller become responsible for the FileIO pointer
Expand All @@ -200,17 +202,9 @@ namespace API

if(!newFileIO->isOpened())
{
newFileIO.reset();
throw(Kernel::Exception::FileError("Can not open target file for filebased box controller ",fileName));
}

// kill old fileIO if any
if(this->m_fileIO)// should happen in destructor anyway but just to be carefull about it
{
this->m_fileIO->flushCache();
this->m_fileIO->closeFile();
}

this->m_fileIO = newFileIO;
}

Expand Down
72 changes: 41 additions & 31 deletions Code/Mantid/Framework/MDAlgorithms/src/PlusMD.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ namespace MDAlgorithms
size_t initial_numEvents = ws1->getNPoints();

// Make a leaf-only iterator through all boxes with events in the RHS workspace
//TODO: OMP
MDBoxIterator<MDE,nd> it2(box2, 1000, true);
do
{
Expand All @@ -123,37 +124,46 @@ namespace MDAlgorithms
prog2->resetNumSteps( ts->size(), 0.4, 0.6);
tp.joinAll();

// Now we need to save all the data that was not saved before.
if (ws1->isFileBacked())
{
// flusush disk kernel buffer and save all still in memory
ws1->getBoxController()->getFileIO()->flushCache();
//// Flush anything else in the to-write buffer
//BoxController_sptr bc = ws1->getBoxController();

//prog.resetNumSteps(bc->getTotalNumMDBoxes(), 0.6, 1.0);
//MDBoxIterator<MDE,nd> it1(box1, 1000, true);
//while (true)
//{
// MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it1.getBox());
// if (box)
// {
// // Something was maybe added to this box
// if (box->getEventVectorSize() > 0)
// {
// // By getting the events, this will merge the newly added and the cached events.
// box->getEvents();
// // The MRU to-write cache will optimize writes by reducing seek times
// box->releaseEvents();
// }
// }
// prog.report("Saving");
// if (!it1.next()) break;
//}
//bc->getDiskBuffer().flushCache();
// Flush the data writes to disk.
box1->getISaveable()->flushData();
}
//// Now we need to save all the data that was not saved before.
//if (ws1->isFileBacked())
//{
// // flusush disk kernel buffer and save all still in memory
// ws1->getBoxController()->getFileIO()->flushCache();
// // Flush the data writes to disk from nexus IO buffer
// ws1->getBoxController()->getFileIO()->flushData();
//}
//if(ws2->isFileBacked())
//{
// // flusush disk kernel buffer and save all still in memory
// ws2->getBoxController()->getFileIO()->flushCache();
// // Flush the data writes to disk from nexus IO buffer
// ws2->getBoxController()->getFileIO()->flushData();

// //// Flush anything else in the to-write buffer
// //BoxController_sptr bc = ws1->getBoxController();

// //prog.resetNumSteps(bc->getTotalNumMDBoxes(), 0.6, 1.0);
// //MDBoxIterator<MDE,nd> it1(box1, 1000, true);
// //while (true)
// //{
// // MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it1.getBox());
// // if (box)
// // {
// // // Something was maybe added to this box
// // if (box->getEventVectorSize() > 0)
// // {
// // // By getting the events, this will merge the newly added and the cached events.
// // box->getEvents();
// // // The MRU to-write cache will optimize writes by reducing seek times
// // box->releaseEvents();
// // }
// // }
// // prog.report("Saving");
// // if (!it1.next()) break;
// //}
// //bc->getDiskBuffer().flushCache();

//}

this->progress(0.95, "Refreshing cache");
ws1->refreshCache();
Expand Down

0 comments on commit 3139d65

Please sign in to comment.