Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/bugfix/8325_load_log_bug'
Browse files Browse the repository at this point in the history
  • Loading branch information
@NickDraper
NickDraper committed Nov 1, 2013
2 parents e44c896 + 2649ad7 commit 9b3ac49
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 227 deletions.
2 changes: 1 addition & 1 deletion Code/Mantid/Framework/DataHandling/inc/MantidDataHandling/LoadLog.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ namespace Mantid
void loadTwoColumnLogFile(std::ifstream& logFileStream, std::string logFileName, API::Run& run);

/// Returns the number of columns in the log file.
int countNumberColumns(std::ifstream& logFileStream);
int countNumberColumns(std::ifstream& logFileStream, const std::string& logFileName);

/// TimeSeriesProperty<int> containing data periods. Created by LogParser
boost::shared_ptr<Kernel::Property> m_periods;
Expand Down
83 changes: 68 additions & 15 deletions Code/Mantid/Framework/DataHandling/src/LoadLog.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ namespace Mantid
"This must be one fewer than the number of columns in the file. "
"Optional: leave blank for no units in any log.");

declareProperty("NumberOfColumns", Mantid::EMPTY_INT(), "Number of columns in the file. If not set Mantid will attempt to guess.");
}

/**
Expand Down Expand Up @@ -147,24 +148,24 @@ namespace Mantid
throw std::invalid_argument("More than one log name provided. Invalid ISIS log file.");
}

try
{
loadThreeColumnLogFile(logFileStream, m_filename, localWorkspace->mutableRun());
return;
}
catch(std::exception&)
{
}
int colNum = static_cast<int>(getProperty("NumberOfColumns"));

try
if(colNum == Mantid::EMPTY_INT())
{
// Since we passed the stream to "loadThreeColumnLogFile" above, we need to reset position to start.
logFileStream.seekg(0);
loadTwoColumnLogFile(logFileStream, extractLogName(names), localWorkspace->mutableRun());
colNum = countNumberColumns(logFileStream, m_filename);
}
catch(std::exception&)

switch(colNum)
{
throw std::invalid_argument("The log file provided is invalid as it has more than three columns.");
case 2:
loadTwoColumnLogFile(logFileStream, extractLogName(names), localWorkspace->mutableRun());
break;
case 3:
loadThreeColumnLogFile(logFileStream, m_filename, localWorkspace->mutableRun());
break;
default:
throw std::invalid_argument("The log file provided is invalid as it has less than 2 or more than three columns.");
break;
}
}

Expand Down Expand Up @@ -273,7 +274,7 @@ namespace Mantid

if ( LoadLog::string != l_kind && LoadLog::number != l_kind)
{
throw std::invalid_argument("ISIS log file contains unrecognised third column entries: " + logFileName);
continue; //no value defined, just skip this entry
}

// column two in .log file is called block column
Expand Down Expand Up @@ -537,5 +538,57 @@ namespace Mantid
return true;
}

/**
* Count the number of columns in the first line of the text file
* @param logFileStream :: stream to the file
*/
int LoadLog::countNumberColumns(std::ifstream& logFileStream, const std::string& logFileName)
{
if (!logFileStream)
{
throw std::invalid_argument("Unable to open file " + m_filename);
}

std::string str;
kind l_kind(LoadLog::empty);

//extract first line of file
Mantid::Kernel::extractToEOL(logFileStream,str);

if ( !isDateTimeString(str) )
{
throw std::invalid_argument("File" + logFileName + " is not a standard ISIS log file. Expected to be a file starting with DateTime String format.");
}

std::stringstream line(str);
std::string timecolumn;
line >> timecolumn;

std::string blockcolumn;
line >> blockcolumn;
l_kind = classify(blockcolumn);

if ( LoadLog::string != l_kind && LoadLog::number != l_kind )
{
throw std::invalid_argument("ISIS log file contains unrecognised second column entries:" + logFileName);
}

std::string valuecolumn;
line >> valuecolumn;
l_kind = classify(valuecolumn);

//reset file back to the beginning
logFileStream.seekg(0);

if ( LoadLog::string != l_kind && LoadLog::number != l_kind)
{
return 2; //looks like a two column file
}
else
{
return 3; //looks like a three column file
}
}

} // namespace DataHandling
} // namespace Mantid
24 changes: 24 additions & 0 deletions Code/Mantid/Framework/DataHandling/src/LoadRawHelper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "MantidDataHandling/LoadLog.h"
#include "MantidDataHandling/LoadAscii.h"

#include <boost/algorithm/string/predicate.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/shared_ptr.hpp>
#include <Poco/File.h>
Expand Down Expand Up @@ -670,14 +671,37 @@ namespace Mantid
std::list<std::string>::const_iterator logPath;
for (logPath = logFiles.begin(); logPath != logFiles.end(); ++logPath)
{
//check for log files we should just ignore
std::string ignoreSuffix = "ICPstatus.txt";
if(boost::algorithm::ends_with(*logPath, ignoreSuffix))
{
g_log.information("Skipping log file: " + *logPath);
continue;
}

ignoreSuffix = "ICPdebug.txt";
if(boost::algorithm::ends_with(*logPath, ignoreSuffix))
{
g_log.information("Skipping log file: " + *logPath);
continue;
}

// Create a new object for each log file.
IAlgorithm_sptr loadLog = createChildAlgorithm("LoadLog");

// Pass through the same input filename
loadLog->setPropertyValue("Filename", *logPath);
// Set the workspace property to be the same one filled above
loadLog->setProperty<MatrixWorkspace_sptr> ("Workspace", localWorkspace);
// Pass the name of the log file explicitly to LoadLog.
loadLog->setPropertyValue("Names", extractLogName(*logPath));

//Force loading two column file if it's an ISIS ICPevent log
if(boost::algorithm::ends_with(*logPath, "ICPevent.txt"))
{
loadLog->setPropertyValue("NumberOfColumns", "2");
}

// Enable progress reporting by Child Algorithm - if progress range has duration
if ( progStart < progEnd )
{
Expand Down
211 changes: 0 additions & 211 deletions Code/Mantid/Framework/DataHandling/test/RemoveLogsTest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,217 +87,6 @@ class RemoveLogsTest : public CxxTest::TestSuite
}


void testExecWithRawDatafile()
{
FrameworkManager::Instance();
//if ( !loader.isInitialized() ) loader.initialize();

LoadLog loaderRawFile;
loaderRawFile.initialize();

// Path to test input file assumes Test directory checked out from SVN
loaderRawFile.setPropertyValue("Filename", "HRP37125.raw");
inputFile = loaderRawFile.getPropertyValue("Filename");

outputSpace = "RemoveLogsTestraw-datafile";
// Create an empty workspace and put it in the AnalysisDataService
Workspace_sptr ws = WorkspaceFactory::Instance().create("Workspace2D",1,1,1);

TS_ASSERT_THROWS_NOTHING(AnalysisDataService::Instance().add(outputSpace, ws));
loaderRawFile.setPropertyValue("Workspace", outputSpace);

std::string result;
TS_ASSERT_THROWS_NOTHING( result = loaderRawFile.getPropertyValue("Filename") )
TS_ASSERT( ! result.compare(inputFile));

TS_ASSERT_THROWS_NOTHING( result = loaderRawFile.getPropertyValue("Workspace") )

TS_ASSERT( ! result.compare(outputSpace));


TS_ASSERT_THROWS_NOTHING(loaderRawFile.execute());

TS_ASSERT( loaderRawFile.isExecuted() );

// Get back the saved workspace
MatrixWorkspace_sptr output;
TS_ASSERT_THROWS_NOTHING(output = AnalysisDataService::Instance().retrieveWS<MatrixWorkspace>(outputSpace));

if ( !remover.isInitialized() ) remover.initialize();
TS_ASSERT_THROWS_NOTHING(remover.setPropertyValue("Workspace", outputSpace))
TS_ASSERT_THROWS_NOTHING(remover.execute());


TS_ASSERT( remover.isExecuted() );

// logs should have been removed
TS_ASSERT_THROWS( output->run().getLogData("ICPevent"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("cphs_6"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("PROP3"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("SE_He_Level"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("TEMP1"), std::runtime_error);

AnalysisDataService::Instance().remove(outputSpace);
}


// Same idea as testExecWithRawDataFile() but testing on a raw file with the extension .s#
// where # is some integer ranging from 01,02,...,99 I believe
void testExecWithRawDatafile_s_type()
{
//if ( !loader.isInitialized() ) loader.initialize();

LoadLog loaderRawFile;
loaderRawFile.initialize();

// Path to test input file assumes Test directory checked out from SVN
TS_ASSERT_THROWS_NOTHING( loaderRawFile.setPropertyValue("Filename", "CSP74683.s02") )
inputFile = loaderRawFile.getPropertyValue("Filename");

outputSpace = "RemoveLogsTest-rawdatafile_so_type";
TS_ASSERT_THROWS( loaderRawFile.setPropertyValue("Workspace", outputSpace), std::invalid_argument)
// Create an empty workspace and put it in the AnalysisDataService
Workspace_sptr ws = WorkspaceFactory::Instance().create("Workspace2D",1,1,1);

TS_ASSERT_THROWS_NOTHING(AnalysisDataService::Instance().add(outputSpace, ws));

std::string result;
TS_ASSERT_THROWS_NOTHING( result = loaderRawFile.getPropertyValue("Filename") )
TS_ASSERT( ! result.compare(inputFile));

TS_ASSERT_THROWS_NOTHING( result = loaderRawFile.getPropertyValue("Workspace") )
TS_ASSERT( ! result.compare(outputSpace));


TS_ASSERT_THROWS_NOTHING(loaderRawFile.execute());

TS_ASSERT( loaderRawFile.isExecuted() );

// Get back the saved workspace
MatrixWorkspace_sptr output;
TS_ASSERT_THROWS_NOTHING(output = AnalysisDataService::Instance().retrieveWS<MatrixWorkspace>(outputSpace));

if ( !remover.isInitialized() ) remover.initialize();
TS_ASSERT_THROWS_NOTHING(remover.setPropertyValue("Workspace", outputSpace))
TS_ASSERT_THROWS_NOTHING(remover.execute());


TS_ASSERT( remover.isExecuted() );

// logs should have been removed
TS_ASSERT_THROWS( output->run().getLogData("ICPevent"), std::runtime_error);

AnalysisDataService::Instance().remove(outputSpace);
}

void testExecWiththreecolumnLogfile()
{
FrameworkManager::Instance();
//if ( !loader.isInitialized() ) loader.initialize();

LoadLog loaderRawFile;
loaderRawFile.initialize();

// Path to test input file assumes Test directory checked out from SVN
loaderRawFile.setPropertyValue("Filename", "NIMROD00001097.raw");
inputFile = loaderRawFile.getPropertyValue("Filename");

outputSpace = "threecoulmlog_datafile";
// Create an empty workspace and put it in the AnalysisDataService
Workspace_sptr ws = WorkspaceFactory::Instance().create("Workspace2D",1,1,1);

TS_ASSERT_THROWS_NOTHING(AnalysisDataService::Instance().add(outputSpace, ws));
loaderRawFile.setPropertyValue("Workspace", outputSpace);

std::string result;
TS_ASSERT_THROWS_NOTHING( result = loaderRawFile.getPropertyValue("Filename") )
TS_ASSERT( ! result.compare(inputFile));

TS_ASSERT_THROWS_NOTHING( result = loaderRawFile.getPropertyValue("Workspace") )

TS_ASSERT( ! result.compare(outputSpace));


TS_ASSERT_THROWS_NOTHING(loaderRawFile.execute());

TS_ASSERT( loaderRawFile.isExecuted() );

// Get back the saved workspace
MatrixWorkspace_sptr output;
TS_ASSERT_THROWS_NOTHING(output = AnalysisDataService::Instance().retrieveWS<MatrixWorkspace>(outputSpace));

if ( !remover.isInitialized() ) remover.initialize();
TS_ASSERT_THROWS_NOTHING(remover.setPropertyValue("Workspace", outputSpace))
TS_ASSERT_THROWS_NOTHING(remover.execute());


TS_ASSERT( remover.isExecuted() );

// logs should have been removed
TS_ASSERT_THROWS( output->run().getLogData("ICPevent"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("J6CX"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("BeamCurrent"), std::runtime_error);

AnalysisDataService::Instance().remove(outputSpace);


}

void test_withAlternateDatastream()
{
FrameworkManager::Instance();
//if ( !loader.isInitialized() ) loader.initialize();

LoadLog loaderRawFile;
loaderRawFile.initialize();

// Path to test input file assumes Test directory checked out from SVN
loaderRawFile.setPropertyValue("Filename", "OFFSPEC00004622.raw");
inputFile = loaderRawFile.getPropertyValue("Filename");

outputSpace = "ads_datafile";
// Create an empty workspace and put it in the AnalysisDataService
Workspace_sptr ws = WorkspaceFactory::Instance().create("Workspace2D",1,1,1);

TS_ASSERT_THROWS_NOTHING(AnalysisDataService::Instance().add(outputSpace, ws));
loaderRawFile.setPropertyValue("Workspace", outputSpace);

std::string result;
TS_ASSERT_THROWS_NOTHING( result = loaderRawFile.getPropertyValue("Filename") )
TS_ASSERT( ! result.compare(inputFile));

TS_ASSERT_THROWS_NOTHING( result = loaderRawFile.getPropertyValue("Workspace") )

TS_ASSERT( ! result.compare(outputSpace));


TS_ASSERT_THROWS_NOTHING(loaderRawFile.execute());

TS_ASSERT( loaderRawFile.isExecuted() );

// Get back the saved workspace
MatrixWorkspace_sptr output;
TS_ASSERT_THROWS_NOTHING(output = AnalysisDataService::Instance().retrieveWS<MatrixWorkspace>(outputSpace));

if ( !remover.isInitialized() ) remover.initialize();
TS_ASSERT_THROWS_NOTHING(remover.setPropertyValue("Workspace", outputSpace))
TS_ASSERT_THROWS_NOTHING(remover.execute());


TS_ASSERT( remover.isExecuted() );

// logs should have been removed
TS_ASSERT_THROWS( output->run().getLogData("ICPevent"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("RF1Ampon"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("ShutterStatus"), std::runtime_error);
TS_ASSERT_THROWS( output->run().getLogData("b2v2"), std::runtime_error);

AnalysisDataService::Instance().remove(outputSpace);


}


void do_test_SNSTextFile(std::string names, std::string units, bool willFail, bool createWorkspace = true)
{
// Create an empty workspace and put it in the AnalysisDataService
Expand Down

0 comments on commit 9b3ac49

Please sign in to comment.