-
Notifications
You must be signed in to change notification settings - Fork 122
/
LoadPreNexus.cpp
325 lines (289 loc) · 12.6 KB
/
LoadPreNexus.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
// NScD Oak Ridge National Laboratory, European Spallation Source,
// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidDataHandling/LoadPreNexus.h"
#include "MantidAPI/FileProperty.h"
#include "MantidAPI/IEventWorkspace.h"
#include "MantidAPI/RegisterFileLoader.h"
#include "MantidAPI/Run.h"
#include "MantidAPI/WorkspaceOpOverloads.h"
#include "MantidDataHandling/LoadEventNexus.h"
#include "MantidDataHandling/LoadTOFRawNexus.h"
#include "MantidKernel/BoundedValidator.h"
#include "MantidKernel/ListValidator.h"
#include "MantidKernel/System.h"
#include "MantidKernel/VisibleWhenProperty.h"
#include <Poco/DOM/AutoPtr.h>
#include <Poco/DOM/DOMParser.h>
#include <Poco/DOM/Document.h>
#include <Poco/DOM/Element.h>
#include <Poco/DOM/NodeFilter.h>
#include <Poco/DOM/NodeIterator.h>
#include <Poco/DOM/NodeList.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Poco/SAX/InputSource.h>
#include <exception>
#include <fstream>
using namespace Mantid::Kernel;
using namespace Mantid::API;
using std::size_t;
using std::string;
using std::vector;
namespace Mantid {
namespace DataHandling {
DECLARE_FILELOADER_ALGORITHM(LoadPreNexus)
static const string RUNINFO_PARAM("Filename");
static const string MAP_PARAM("MappingFilename");
/// @copydoc Mantid::API::IAlgorithm::name()
const std::string LoadPreNexus::name() const { return "LoadPreNexus"; }
/// @copydoc Mantid::API::IAlgorithm::version()
int LoadPreNexus::version() const { return 1; }
/// @copydoc Mantid::API::IAlgorithm::category()
const std::string LoadPreNexus::category() const { return "DataHandling\\PreNexus;Workflow\\DataHandling"; }
/**
* Return the confidence with with this algorithm can load the file
* @param descriptor A descriptor for the file
* @returns An integer specifying the confidence level. 0 indicates it will not
* be used
*/
int LoadPreNexus::confidence(Kernel::FileDescriptor &descriptor) const {
const std::string &filename = descriptor.filename();
if (filename.size() > 12 ? (filename.compare(filename.size() - 12, 12, "_runinfo.xml") == 0) : false)
return 80;
else
return 0;
}
/// @copydoc Mantid::API::Algorithm::init()
void LoadPreNexus::init() {
// runfile to read in
declareProperty(std::make_unique<FileProperty>(RUNINFO_PARAM, "", FileProperty::Load, "_runinfo.xml"),
"The name of the runinfo file to read, including its full or relative "
"path.");
// copied (by hand) from LoadEventPreNexus2
declareProperty(std::make_unique<FileProperty>(MAP_PARAM, "", FileProperty::OptionalLoad, ".dat"),
"File containing the pixel mapping (DAS pixels to pixel IDs) file "
"(typically INSTRUMENT_TS_YYYY_MM_DD.dat). The filename will be found "
"automatically if not specified.");
auto mustBePositive = std::make_shared<BoundedValidator<int>>();
mustBePositive->setLower(1);
declareProperty("ChunkNumber", EMPTY_INT(), mustBePositive,
"If loading the file by sections ('chunks'), this is the "
"section number of this execution of the algorithm.");
declareProperty("TotalChunks", EMPTY_INT(), mustBePositive,
"If loading the file by sections ('chunks'), this is the "
"total number of sections.");
// TotalChunks is only meaningful if ChunkNumber is set
// Would be nice to be able to restrict ChunkNumber to be <= TotalChunks at
// validation
setPropertySettings("TotalChunks", std::make_unique<VisibleWhenProperty>("ChunkNumber", IS_NOT_DEFAULT));
std::vector<std::string> propOptions{"Auto", "Serial", "Parallel"};
declareProperty("UseParallelProcessing", "Auto", std::make_shared<StringListValidator>(propOptions),
"Use multiple cores for loading the data?\n"
" Auto: Use serial loading for small data sets, parallel "
"for large data sets.\n"
" Serial: Use a single core.\n"
" Parallel: Use all available cores.");
declareProperty(std::make_unique<PropertyWithValue<bool>>("LoadMonitors", true, Direction::Input),
"Load the monitors from the file.");
declareProperty(std::make_unique<WorkspaceProperty<>>("OutputWorkspace", "", Direction::Output),
"An output workspace.");
}
//----------------------------------------------------------------------------------------------
/// @copydoc Mantid::API::Algorithm::exec()
void LoadPreNexus::exec() {
string runinfo = this->getPropertyValue(RUNINFO_PARAM);
string mapfile = this->getPropertyValue(MAP_PARAM);
int chunkTotal = this->getProperty("TotalChunks");
int chunkNumber = this->getProperty("ChunkNumber");
if (isEmpty(chunkTotal) || isEmpty(chunkNumber)) {
chunkNumber = EMPTY_INT();
chunkTotal = EMPTY_INT();
} else {
if (chunkNumber > chunkTotal)
throw std::out_of_range("ChunkNumber cannot be larger than TotalChunks");
}
string useParallel = this->getProperty("UseParallelProcessing");
string wsname = this->getProperty("OutputWorkspace");
bool loadmonitors = this->getProperty("LoadMonitors");
// determine the event file names
Progress prog(this, 0., .1, 1);
vector<string> eventFilenames;
string dataDir;
this->parseRuninfo(runinfo, dataDir, eventFilenames);
prog.doReport("parsed runinfo file");
// do math for the progress bar
size_t numFiles = eventFilenames.size() + 1; // extra 1 is nexus logs
if (loadmonitors)
numFiles++;
double prog_start = .1;
double prog_delta = (1. - prog_start) / static_cast<double>(numFiles);
// load event files
string temp_wsname;
for (size_t i = 0; i < eventFilenames.size(); i++) {
if (i == 0)
temp_wsname = wsname;
else
temp_wsname = "__" + wsname + "_temp__";
auto alg = createChildAlgorithm("LoadEventPreNexus", prog_start, prog_start + prog_delta);
alg->setProperty("EventFilename", dataDir + eventFilenames[i]);
alg->setProperty("MappingFilename", mapfile);
alg->setProperty("ChunkNumber", chunkNumber);
alg->setProperty("TotalChunks", chunkTotal);
alg->setProperty("UseParallelProcessing", useParallel);
alg->setPropertyValue("OutputWorkspace", temp_wsname);
alg->executeAsChildAlg();
prog_start += prog_delta;
if (i == 0) {
m_outputWorkspace = alg->getProperty("OutputWorkspace");
} else {
IEventWorkspace_sptr tempws = alg->getProperty("OutputWorkspace");
// clean up properties before adding data
Run &run = tempws->mutableRun();
if (run.hasProperty("gd_prtn_chrg"))
run.removeProperty("gd_prtn_chrg");
if (run.hasProperty("proton_charge"))
run.removeProperty("proton_charge");
m_outputWorkspace += tempws;
}
}
// load the logs
this->runLoadNexusLogs(runinfo, dataDir, prog_start, prog_start + prog_delta);
prog_start += prog_delta;
// publish output workspace
this->setProperty("OutputWorkspace", m_outputWorkspace);
// load the monitor
if (loadmonitors) {
this->runLoadMonitors(prog_start, 1.);
}
}
/**
* Parse the runinfo file to find the names of the neutron event files.
*
* @param runinfo Runinfo file with full path.
* @param dataDir Directory where the runinfo file lives.
* @param eventFilenames vector of all possible event files. This is filled by
*the algorithm.
*/
void LoadPreNexus::parseRuninfo(const string &runinfo, string &dataDir, vector<string> &eventFilenames) {
eventFilenames.clear();
// Create a Poco Path object for runinfo filename
Poco::Path runinfoPath(runinfo, Poco::Path::PATH_GUESS);
// Now lets get the directory
Poco::Path dirPath(runinfoPath.parent());
dataDir = dirPath.absolute().toString();
g_log.debug() << "Data directory \"" << dataDir << "\"\n";
std::ifstream in(runinfo.c_str());
Poco::XML::InputSource src(in);
Poco::XML::DOMParser parser;
Poco::AutoPtr<Poco::XML::Document> pDoc = parser.parse(&src);
Poco::XML::NodeIterator it(pDoc, Poco::XML::NodeFilter::SHOW_ELEMENT);
Poco::XML::Node *pNode = it.nextNode(); // root node
while (pNode) {
if (pNode->nodeName() == "RunInfo") // standard name for this type
{
pNode = pNode->firstChild();
while (pNode) {
if (pNode->nodeName() == "FileList") {
pNode = pNode->firstChild();
while (pNode) {
if (pNode->nodeName() == "DataList") {
pNode = pNode->firstChild();
while (pNode) {
if (pNode->nodeName() == "scattering") {
auto *element = static_cast<Poco::XML::Element *>(pNode);
eventFilenames.emplace_back(element->getAttribute("name"));
}
pNode = pNode->nextSibling();
}
} else // search for DataList
pNode = pNode->nextSibling();
}
} else // search for FileList
pNode = pNode->nextSibling();
}
} else // search for RunInfo
pNode = pNode->nextSibling();
}
// report the results to the log
if (eventFilenames.size() == 1) {
g_log.debug() << "Found 1 event file: \"" << eventFilenames[0] << "\"\n";
} else {
g_log.debug() << "Found " << eventFilenames.size() << " event files:";
for (auto &eventFilename : eventFilenames) {
g_log.debug() << "\"" << eventFilename << "\" ";
}
g_log.debug() << "\n";
}
}
/**
* Load logs from a nexus file onto the workspace.
*
* @param runinfo Runinfo file with full path.
* @param dataDir Directory where the runinfo file lives.
* @param prog_start Starting position for the progress bar.
* @param prog_stop Ending position for the progress bar.
*/
void LoadPreNexus::runLoadNexusLogs(const string &runinfo, const string &dataDir, const double prog_start,
const double prog_stop) {
// determine the name of the file "inst_run"
string shortName = runinfo.substr(runinfo.find_last_of("/\\") + 1);
shortName = shortName.substr(0, shortName.find("_runinfo.xml"));
g_log.debug() << "SHORTNAME = \"" << shortName << "\"\n";
// put together a list of possible locations
vector<string> possibilities;
possibilities.emplace_back(dataDir + shortName + "_event.nxs"); // next to runinfo
possibilities.emplace_back(dataDir + shortName + "_histo.nxs");
possibilities.emplace_back(dataDir + shortName + ".nxs");
possibilities.emplace_back(dataDir + "../NeXus/" + shortName + "_event.nxs"); // in NeXus directory
possibilities.emplace_back(dataDir + "../NeXus/" + shortName + "_histo.nxs");
possibilities.emplace_back(dataDir + "../NeXus/" + shortName + ".nxs");
// run the algorithm
bool loadedLogs = false;
for (auto &possibility : possibilities) {
if (Poco::File(possibility).exists()) {
g_log.information() << "Loading logs from \"" << possibility << "\"\n";
auto alg = createChildAlgorithm("LoadNexusLogs", prog_start, prog_stop);
alg->setProperty("Workspace", m_outputWorkspace);
alg->setProperty("Filename", possibility);
alg->setProperty("OverwriteLogs", false);
alg->executeAsChildAlg();
loadedLogs = true;
// Reload instrument so SNAP can use log values
std::string entry_name = LoadTOFRawNexus::getEntryName(possibility);
LoadEventNexus::runLoadInstrument(possibility, m_outputWorkspace, entry_name, this);
break;
}
}
if (!loadedLogs)
g_log.notice() << "Did not find a nexus file to load logs from\n";
}
/**
* Load the monitor files.
*
* @param prog_start Starting position for the progress bar.
* @param prog_stop Ending position for the progress bar.
*/
void LoadPreNexus::runLoadMonitors(const double prog_start, const double prog_stop) {
std::string mon_wsname = this->getProperty("OutputWorkspace");
mon_wsname.append("_monitors");
try {
auto alg = createChildAlgorithm("LoadPreNexusMonitors", prog_start, prog_stop);
alg->setPropertyValue("RunInfoFilename", this->getProperty(RUNINFO_PARAM));
alg->setPropertyValue("OutputWorkspace", mon_wsname);
alg->executeAsChildAlg();
MatrixWorkspace_sptr mons = alg->getProperty("OutputWorkspace");
this->declareProperty(std::make_unique<WorkspaceProperty<>>("MonitorWorkspace", mon_wsname, Direction::Output),
"Monitors from the Event NeXus file");
this->setProperty("MonitorWorkspace", mons);
// Add an internal pointer to monitor workspace in the 'main' workspace
m_outputWorkspace->setMonitorWorkspace(mons);
} catch (std::exception &e) {
g_log.warning() << "Failed to load monitors: " << e.what() << "\n";
}
}
} // namespace DataHandling
} // namespace Mantid