Skip to content

Commit

Permalink
Refs #1661: Fixes to the splitter and filter by log value code; more …
Browse files Browse the repository at this point in the history 
…tests added.
  • Loading branch information
Janik Zikovsky committed Sep 17, 2010
1 parent 0d4007f commit b9d9c3e
Show file tree
Hide file tree
Showing 5 changed files with 215 additions and 15 deletions.
10 changes: 9 additions & 1 deletion Code/Mantid/Algorithms/src/FilterByLogValue.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,22 @@ void FilterByLogValue::exec()
log->makeFilterByValue(splitter, min, max, tolerance);
}

g_log.information() << splitter.size() << " entries in the filter.\n";

// if (splitter.size() > 0)
// {
// PulseTimeType first_time = splitter[0].start();
// for (size_t i=0; i<splitter.size(); i++)
// std::cout << i << ": " << (splitter[i].start()-first_time) << " to " << (splitter[i].stop()-first_time) << "\n";
// }

int numberOfSpectra = inputWS->getNumberHistograms();

// Initialise the progress reporting object
Progress prog(this,0.0,1.0,numberOfSpectra);

// Loop over the histograms (detector spectra)
//PARALLEL_FOR_NO_WSP_CHECK()
PARALLEL_FOR_NO_WSP_CHECK()
for (int i = 0; i < numberOfSpectra; ++i)
{
PARALLEL_START_INTERUPT_REGION
Expand Down
72 changes: 72 additions & 0 deletions Code/Mantid/Algorithms/test/FilterByLogValueTest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "MantidAlgorithms/FilterByLogValue.h"
#include "MantidDataHandling/LoadEventPreNeXus.h"
#include "MantidKernel/DateAndTime.h"
#include "MantidKernel/TimeSeriesProperty.h"
#include "MantidDataObjects/EventWorkspace.h"

using namespace Mantid::Algorithms;
Expand Down Expand Up @@ -46,6 +47,7 @@ class FilterByLogValueTest : public CxxTest::TestSuite
}



void testExec()
{
std::string outputWS;
Expand Down Expand Up @@ -92,7 +94,77 @@ class FilterByLogValueTest : public CxxTest::TestSuite

//Proton charge is lower
TS_ASSERT_LESS_THAN( outWS->run().getProtonCharge(), WS->run().getProtonCharge() );
}





void setUp_Event2()
{
inputWS = "eventWS2";
LoadEventPreNeXus loader;
loader.initialize();
std::string eventfile( "../../../../Test/Data/sns_event_prenexus/CNCS_7850_neutron_event.dat" );
std::string pulsefile( "../../../../Test/Data/sns_event_prenexus/CNCS_7850_pulseid.dat" );
loader.setPropertyValue("EventFilename", eventfile);
loader.setProperty("PulseidFilename", pulsefile);
loader.setPropertyValue("MappingFilename", "../../../../Test/Data/sns_event_prenexus/CNCS_TS_2008_08_18.dat");
loader.setPropertyValue("OutputWorkspace", inputWS);
loader.execute();
TS_ASSERT (loader.isExecuted() );
}

void xtestExec2_slow()
{
std::string outputWS;
this->setUp_Event2();

//Retrieve Workspace
WS = boost::dynamic_pointer_cast<EventWorkspace>(AnalysisDataService::Instance().retrieve(inputWS));
TS_ASSERT( WS ); //workspace is loaded
size_t start_blocksize = WS->blocksize();
size_t num_events = WS->getNumberEvents();

//Do the filtering now.
FilterByLogValue * alg = new FilterByLogValue();
alg->initialize();
alg->setPropertyValue("InputWorkspace", inputWS);
outputWS = "eventWS_relative";
alg->setPropertyValue("OutputWorkspace", outputWS);
alg->setPropertyValue("LogName", "ProtonCharge");
//We set the minimum high enough to cut out some real charge too, not just zeros.
alg->setPropertyValue("MinimumValue", "5e6");
alg->setPropertyValue("MaximumValue", "1e20");
alg->setPropertyValue("TimeTolerance", "3e-3");

alg->execute();
TS_ASSERT( alg->isExecuted() );

//Retrieve Workspace changed
EventWorkspace_sptr outWS;
outWS = boost::dynamic_pointer_cast<EventWorkspace>(AnalysisDataService::Instance().retrieve(outputWS));
TS_ASSERT( outWS ); //workspace is loaded

//Things that haven't changed
TS_ASSERT_EQUALS( outWS->blocksize(), WS->blocksize());
TS_ASSERT_EQUALS( outWS->getNumberHistograms(), WS->getNumberHistograms());

//There should be some events
TS_ASSERT_LESS_THAN( 0, outWS->getNumberEvents());

// Not many events left: 34612
TS_ASSERT_LESS_THAN( outWS->getNumberEvents(), WS->getNumberEvents());
TS_ASSERT_DELTA( outWS->getNumberEvents() , 1093284, 100);

//Proton charge is lower
TS_ASSERT_LESS_THAN( outWS->run().getProtonCharge(), WS->run().getProtonCharge() );

//Check the log entries
TimeSeriesProperty<double> * log = dynamic_cast<TimeSeriesProperty<double> * >(outWS->run().getProperty("ProtonCharge"));
TS_ASSERT(log);
for (std::size_t i=0; i<log->realSize(); i++)
TS_ASSERT_LESS_THAN( 0, log->nthValue(i));

}

Expand Down
24 changes: 24 additions & 0 deletions Code/Mantid/DataObjects/test/EventListTest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,7 @@ class EventListTest : public CxxTest::TestSuite
TS_ASSERT_EQUALS( el.getNumberEvents(), 3000);
}

//-----------------------------------------------------------------------------------------------
void testSetTofs()
{
this->fake_data();
Expand All @@ -484,6 +485,7 @@ class EventListTest : public CxxTest::TestSuite
}


//-----------------------------------------------------------------------------------------------
void testFilterByPulseTime()
{
this->fake_data();
Expand Down Expand Up @@ -587,6 +589,28 @@ class EventListTest : public CxxTest::TestSuite
}


//-----------------------------------------------------------------------------------------------
void testSplit_FilterWithOverlap()
{
this->fake_uniform_time_data();

std::vector< EventList * > outputs;
for (int i=0; i<1; i++)
outputs.push_back( new EventList() );

TimeSplitterType split;
split.push_back( SplittingInterval(100, 200, 0) );
split.push_back( SplittingInterval(150, 250, 0) );

//Do the splitting
el.splitByTime(split, outputs);

//No events in the first ouput 0-99
TS_ASSERT_EQUALS( outputs[0]->getNumberEvents(), 150);

}


};


Expand Down
17 changes: 12 additions & 5 deletions Code/Mantid/Kernel/inc/MantidKernel/TimeSeriesProperty.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ class DLLExport TimeSeriesProperty: public Property
//Count the sizes (this function is still stupid and slow and should not be necessary, but I don't know if I should take it out)
for (int i=0; i < numOutputs; i++)
{
TimeSeriesProperty<TYPE> * myOutput = outputs_tsp[index];
TimeSeriesProperty<TYPE> * myOutput = outputs_tsp[i];
if (myOutput)
myOutput->countSize();
}
Expand Down Expand Up @@ -233,14 +233,21 @@ class DLLExport TimeSeriesProperty: public Property
int numgood = 0;
dateAndTime lastTime, t;
PulseTimeType start, stop;
// int count = 0;

for (it = m_propertySeries.begin(); it != m_propertySeries.end(); it++)
{
// count++;

lastTime = t;
//The new entry
t = it->first;
TYPE val = it->second;

// //The time cannot have gone backwards! But it could be the same twice
// if (t < lastTime)
// std::cout << "Time has gone backwards!" << count <<"\n";

//A good value?
isGood = ((val >= min) && (val < max));
if (isGood)
Expand All @@ -267,7 +274,7 @@ class DLLExport TimeSeriesProperty: public Property
else
{
//At least 2 good values. Save the end time
stop = DateAndTime::get_from_absolute_time( t + tol );
stop = DateAndTime::get_from_absolute_time( lastTime + tol );
split.push_back( SplittingInterval(start, stop, 0) );
}
//Reset the number of good ones, for next time
Expand Down Expand Up @@ -370,7 +377,7 @@ class DLLExport TimeSeriesProperty: public Property
*/
std::string setValue(const std::string&)
{
throw Exception::NotImplementedError("Not yet");
throw Exception::NotImplementedError("Not implemented in this class");
}


Expand All @@ -380,7 +387,7 @@ class DLLExport TimeSeriesProperty: public Property
* @param value The associated value
* @return True if insertion successful (i.e. identical time not already in map
*/
bool addValue(const boost::posix_time::ptime &time, const TYPE value)
bool addValue(const Kernel::dateAndTime &time, const TYPE value)
{
m_size++;
return m_propertySeries.insert(typename timeMap::value_type(time, value)) != m_propertySeries.end();
Expand All @@ -405,7 +412,7 @@ class DLLExport TimeSeriesProperty: public Property
*/
bool addValue(const std::time_t &time, const TYPE value)
{
return addValue(boost::posix_time::from_time_t(time), value);
return addValue(Kernel::DateAndTime::from_time_t(time), value);
}

//-----------------------------------------------------------------------------------------------
Expand Down
107 changes: 98 additions & 9 deletions Code/Mantid/Kernel/test/TimeSeriesPropertyTest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite
}


//----------------------------------------------------------------------------
void testAdditionOperator()
{
TimeSeriesProperty<int> * log = new TimeSeriesProperty<int>("MyIntLog");
Expand All @@ -103,10 +104,9 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite
(*log) += log2;

TS_ASSERT_EQUALS( log->size(), 5);


}

//----------------------------------------------------------------------------
void test_filterByTime_and_getTotalValue()
{
TimeSeriesProperty<int> * log = new TimeSeriesProperty<int>("MyIntLog");
Expand All @@ -127,10 +127,49 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite
log->filterByTime(start, stop);
TS_ASSERT_EQUALS( log->realSize(), 3);
TS_ASSERT_EQUALS( log->getTotalValue(), 9);
}



//----------------------------------------------------------------------------
void test_makeFilterByValue()
{
TimeSeriesProperty<double> * log = new TimeSeriesProperty<double>("MyIntLog");
TS_ASSERT( log->addValue("2007-11-30T16:17:00",1) );
TS_ASSERT( log->addValue("2007-11-30T16:17:10",2) );
TS_ASSERT( log->addValue("2007-11-30T16:17:20",3) );
TS_ASSERT( log->addValue("2007-11-30T16:17:30",2.0) );
TS_ASSERT( log->addValue("2007-11-30T16:17:40",2.01) );
TS_ASSERT( log->addValue("2007-11-30T16:17:50",6) );

TS_ASSERT_EQUALS( log->realSize(), 6);

TimeSplitterType splitter;
log->makeFilterByValue(splitter, 1.8, 2.2, 1.0);

TS_ASSERT_EQUALS( splitter.size(), 2);
SplittingInterval s;
dateAndTime t;

s = splitter[0];
t = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:09");
TS_ASSERT_DELTA( s.start(), DateAndTime::get_from_absolute_time(t), 1000);
t = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:11");
TS_ASSERT_DELTA( s.stop(), DateAndTime::get_from_absolute_time(t), 1000);

s = splitter[1];
t = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:29");
TS_ASSERT_DELTA( s.start(), DateAndTime::get_from_absolute_time(t), 1000);
t = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:41");
TS_ASSERT_DELTA( s.stop(), DateAndTime::get_from_absolute_time(t), 1000);


}




//----------------------------------------------------------------------------
void test_splitByTime_and_getTotalValue()
{
TimeSeriesProperty<int> * log = new TimeSeriesProperty<int>("MyIntLog");
Expand All @@ -151,7 +190,7 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite
//Make the outputs
// std::vector< TimeSeriesProperty<int> * > outputs2;
std::vector< Property * > outputs;
for (int i=0; i<4; i++)
for (int i=0; i<5; i++)
{
TimeSeriesProperty<int> * newlog = new TimeSeriesProperty<int>("MyIntLog");
outputs.push_back(newlog);
Expand All @@ -166,27 +205,77 @@ class TimeSeriesPropertyTest : public CxxTest::TestSuite
splitter.push_back( SplittingInterval(start, stop, 0) );

start = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:55");
stop = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:59");
stop = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:56");
splitter.push_back( SplittingInterval(start, stop, 1) );

start = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:56");
stop = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:18:01");
splitter.push_back( SplittingInterval(start, stop, 2) ); //just one entry

start = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:18:09");
stop = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:18:21");
splitter.push_back( SplittingInterval(start, stop, 2) );
splitter.push_back( SplittingInterval(start, stop, 3) );

start = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:18:45");
stop = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:22:50");
splitter.push_back( SplittingInterval(start, stop, 3) );
splitter.push_back( SplittingInterval(start, stop, 4) );

log->splitByTime(splitter, outputs);

TS_ASSERT_EQUALS( dynamic_cast< TimeSeriesProperty<int> * >(outputs[0])->realSize(), 3);
TS_ASSERT_EQUALS( dynamic_cast< TimeSeriesProperty<int> * >(outputs[1])->realSize(), 0);
TS_ASSERT_EQUALS( dynamic_cast< TimeSeriesProperty<int> * >(outputs[2])->realSize(), 2);
TS_ASSERT_EQUALS( dynamic_cast< TimeSeriesProperty<int> * >(outputs[3])->realSize(), 1);
TS_ASSERT_EQUALS( dynamic_cast< TimeSeriesProperty<int> * >(outputs[2])->realSize(), 1);
TS_ASSERT_EQUALS( dynamic_cast< TimeSeriesProperty<int> * >(outputs[3])->realSize(), 2);
TS_ASSERT_EQUALS( dynamic_cast< TimeSeriesProperty<int> * >(outputs[4])->realSize(), 1);

}



//----------------------------------------------------------------------------
void test_splitByTime_withOverlap()
{
TimeSeriesProperty<int> * log = new TimeSeriesProperty<int>("MyIntLog");
TS_ASSERT( log->addValue("2007-11-30T16:17:00",1) );
TS_ASSERT( log->addValue("2007-11-30T16:17:10",2) );
TS_ASSERT( log->addValue("2007-11-30T16:17:20",3) );
TS_ASSERT( log->addValue("2007-11-30T16:17:30",4) );
TS_ASSERT( log->addValue("2007-11-30T16:17:40",5) );
TS_ASSERT( log->addValue("2007-11-30T16:17:50",6) );
TS_ASSERT( log->addValue("2007-11-30T16:18:00",7) );
TS_ASSERT( log->addValue("2007-11-30T16:18:10",8) );
TS_ASSERT( log->addValue("2007-11-30T16:18:20",9) );
TS_ASSERT( log->addValue("2007-11-30T16:18:30",10) );
TS_ASSERT( log->addValue("2007-11-30T16:18:40",11) );
TS_ASSERT( log->addValue("2007-11-30T16:18:50",12) );
TS_ASSERT_EQUALS( log->realSize(), 12);

//Make the outputs
// std::vector< TimeSeriesProperty<int> * > outputs2;
std::vector< Property * > outputs;
for (int i=0; i<1; i++)
{
TimeSeriesProperty<int> * newlog = new TimeSeriesProperty<int>("MyIntLog");
outputs.push_back(newlog);
}

//Make a splitter
dateAndTime start, stop;
TimeSplitterType splitter;
start = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:10");
stop = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:40");
splitter.push_back( SplittingInterval(start, stop, 0) );

start = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:35");
stop = DateAndTime::create_DateAndTime_FromISO8601_String("2007-11-30T16:17:59");
splitter.push_back( SplittingInterval(start, stop, 0) );

log->splitByTime(splitter, outputs);

TS_ASSERT_EQUALS( dynamic_cast< TimeSeriesProperty<int> * >(outputs[0])->realSize(), 5);

}


private:
TimeSeriesProperty<int> *iProp;
TimeSeriesProperty<double> *dProp;
Expand Down

0 comments on commit b9d9c3e

Please sign in to comment.