diff --git a/Code/Mantid/Framework/API/inc/MantidAPI/IMDNode.h b/Code/Mantid/Framework/API/inc/MantidAPI/IMDNode.h index 5cea41bcf918..9205f66d1869 100644 --- a/Code/Mantid/Framework/API/inc/MantidAPI/IMDNode.h +++ b/Code/Mantid/Framework/API/inc/MantidAPI/IMDNode.h @@ -71,6 +71,7 @@ class IMDNode *@param loadFileData -- if true, the data on HDD and not yet in memory are loaded into memory before deleting fileBacked information, if false, all on HDD contents are discarded, which can break the data integrity (used by destructor) */ virtual void clearFileBacked(bool loadFileData)=0; + virtual void reserveMemoryForLoad(uint64_t)=0; /**Save the box at specific disk position using the class, respoinsible for the file IO. */ virtual void saveAt(API::IBoxControllerIO *const /*saver */, uint64_t /*position*/)const=0; diff --git a/Code/Mantid/Framework/MDAlgorithms/src/MergeMDFiles.cpp b/Code/Mantid/Framework/MDAlgorithms/src/MergeMDFiles.cpp index ce5f98266480..2798ea8d4b2d 100644 --- a/Code/Mantid/Framework/MDAlgorithms/src/MergeMDFiles.cpp +++ b/Code/Mantid/Framework/MDAlgorithms/src/MergeMDFiles.cpp @@ -159,17 +159,24 @@ namespace MDAlgorithms TargetBox->clear(); uint64_t nBoxEvents(0); + std::vector numFileEvents(m_EventLoader.size()); + for (size_t iw=0; iwm_EventLoader.size(); iw++) { size_t ID = TargetBox->getID(); + numFileEvents[iw] = static_cast(m_fileComponentsStructure[iw].getEventIndex()[2*ID+1]); + nBoxEvents += numFileEvents[iw]; + } - uint64_t fileLocation = m_fileComponentsStructure[iw].getEventIndex()[2*ID+0]; - size_t numFileEvents = static_cast(m_fileComponentsStructure[iw].getEventIndex()[2*ID+1]); - if(numFileEvents==0)continue; - //TODO: it is possible to avoid the reallocation of the memory at each load - TargetBox->loadAndAddFrom(m_EventLoader[iw],fileLocation,numFileEvents); + // At this point memory required is known, so it is reserved all in one go + TargetBox->reserveMemoryForLoad(nBoxEvents); - nBoxEvents += numFileEvents; + for (size_t iw=0; iwm_EventLoader.size(); iw++) + { + size_t ID = TargetBox->getID(); + uint64_t fileLocation = m_fileComponentsStructure[iw].getEventIndex()[2*ID+0]; + if(numFileEvents[iw]==0) continue; + TargetBox->loadAndAddFrom(m_EventLoader[iw],fileLocation,numFileEvents[iw]); } return nBoxEvents; diff --git a/Code/Mantid/Framework/MDEvents/inc/MantidMDEvents/MDBox.h b/Code/Mantid/Framework/MDEvents/inc/MantidMDEvents/MDBox.h index 347e37008629..ea9bdc625d29 100644 --- a/Code/Mantid/Framework/MDEvents/inc/MantidMDEvents/MDBox.h +++ b/Code/Mantid/Framework/MDEvents/inc/MantidMDEvents/MDBox.h @@ -68,6 +68,7 @@ namespace MDEvents //----------------------------------------------------------------------------------------------- virtual void saveAt(API::IBoxControllerIO *const /* */, uint64_t /*position*/)const; virtual void loadAndAddFrom(API::IBoxControllerIO *const /* */, uint64_t /*position*/, size_t /* Size */); + virtual void reserveMemoryForLoad(uint64_t /* Size */); /**drop events data from memory but keep averages (and file-backed info) */ void clearDataFromMemory(); /** */ diff --git a/Code/Mantid/Framework/MDEvents/inc/MantidMDEvents/MDGridBox.h b/Code/Mantid/Framework/MDEvents/inc/MantidMDEvents/MDGridBox.h index 0058be49fdf2..4d5f63be210c 100644 --- a/Code/Mantid/Framework/MDEvents/inc/MantidMDEvents/MDGridBox.h +++ b/Code/Mantid/Framework/MDEvents/inc/MantidMDEvents/MDGridBox.h @@ -67,6 +67,8 @@ namespace MDEvents /**Load the box data of specified size from the disk location provided using the class, respoinsible for the file IO. */ virtual void loadAndAddFrom(API::IBoxControllerIO *const /* */, uint64_t /*position*/, size_t /* Size */) {/*Not directly loadable */} + virtual void reserveMemoryForLoad(uint64_t /* Size */) + {/*Not directly loadable */} //------------------------------------------------------------------------------------------------------- /** Uses the cached value of points stored in the grid box diff --git a/Code/Mantid/Framework/MDEvents/src/MDBox.cpp b/Code/Mantid/Framework/MDEvents/src/MDBox.cpp index 0860428cd616..ece7dfc1309f 100644 --- a/Code/Mantid/Framework/MDEvents/src/MDBox.cpp +++ b/Code/Mantid/Framework/MDEvents/src/MDBox.cpp @@ -900,9 +900,20 @@ namespace MDEvents FileSaver->saveBlock(TabledData,position); } + + /** + * Reserve all the memory required for loading in one step. + * + * @param size -- number of events to reserve for + */ + TMDE( + void MDBox)::reserveMemoryForLoad(uint64_t size) + { + this->data.reserve(size); + } + /**Load the box data of specified size from the disk location provided using the class, respoinsible for the file IO and append them to exisiting events - * Clear events vector first if overwriting the exisitng events is necessary. The efficiency would be higher if jentle cleaning occurs (the size of event data vector - is nullified but memory still allocated) + * Clear events vector first if overwriting the exisitng events is necessary. * * @param FileSaver -- the pointer to the class, responsible for file IO * @param filePosition -- the place in the direct access file, where necessary data are located @@ -923,9 +934,6 @@ namespace MDEvents std::vector TableData; FileSaver->loadBlock(TableData,filePosition,nEvents); - // convert loaded events to data; - size_t nCurrentEvents = data.size(); - this->data.reserve(nCurrentEvents+nEvents); // convert data to events appending new events to existing MDE::dataToEvents(TableData,data,false); diff --git a/Code/Mantid/Framework/MDEvents/test/MDBoxBaseTest.h b/Code/Mantid/Framework/MDEvents/test/MDBoxBaseTest.h index 14c4a60f1ac7..44733b33da4b 100644 --- a/Code/Mantid/Framework/MDEvents/test/MDBoxBaseTest.h +++ b/Code/Mantid/Framework/MDEvents/test/MDBoxBaseTest.h @@ -49,6 +49,7 @@ class MDBoxBaseTester : public MDBoxBase void setFileBacked(){}; void saveAt(API::IBoxControllerIO *const /* */, uint64_t /*position*/)const{/*Not saveable */}; void loadAndAddFrom(API::IBoxControllerIO *const /* */, uint64_t /*position*/, size_t /* Size */){}; + void reserveMemoryForLoad(uint64_t /* Size */){}; // regardless of what is actually instantiated, base tester would call itself gridbox bool isBox()const{return false;}