-
Notifications
You must be signed in to change notification settings - Fork 122
/
NexusTester.cpp
175 lines (146 loc) · 7.04 KB
/
NexusTester.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
// NScD Oak Ridge National Laboratory, European Spallation Source,
// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidDataHandling/NexusTester.h"
#include "MantidAPI/FileProperty.h"
#include "MantidKernel/CPUTimer.h"
#include "MantidKernel/ListValidator.h"
#include <Poco/File.h>
#include <Poco/Thread.h>
#include <cstdlib>
#include <nexus/NeXusFile.hpp>
using namespace Mantid::Kernel;
using namespace Mantid::API;
namespace Mantid {
namespace DataHandling {
// Register the algorithm into the AlgorithmFactory
DECLARE_ALGORITHM(NexusTester)
//----------------------------------------------------------------------------------------------
/// Algorithm's name for identification. @see Algorithm::name
const std::string NexusTester::name() const { return "NexusTester"; }
/// Algorithm's version for identification. @see Algorithm::version
int NexusTester::version() const { return 1; }
/// Algorithm's category for identification. @see Algorithm::category
const std::string NexusTester::category() const { return "Utility\\Development"; }
//----------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------
/** Initialize the algorithm's properties.
*/
void NexusTester::init() {
std::initializer_list<std::string> exts = {".nxs"};
declareProperty(std::make_unique<FileProperty>("SaveFilename", "", FileProperty::OptionalSave, exts),
"The name of the Nexus file to write.");
declareProperty(std::make_unique<FileProperty>("LoadFilename", "", FileProperty::OptionalLoad, exts),
"The name of the Nexus file to load (optional).\n"
"Must have been written by NexusTester algorithm.");
declareProperty("ChunkSize", 10, "Chunk size for writing/loading, in kb of data");
declareProperty("NumChunks", 10, "Number of chunks to load or write");
declareProperty("Compress", true, "For writing: compress the data.");
declareProperty("HDFCacheSize", 2000000, "HDF cache size, in bytes");
declareProperty("ClearDiskCache", false,
"Clear the linux disk cache before loading.\n"
"Only works on linux AND you need to run MantidPlot in sudo mode (!).");
std::vector<std::string> types{"Zeros", "Incrementing Numbers", "Random Numbers"};
declareProperty("FakeData", "Incrementing Numbers", std::make_shared<StringListValidator>(types),
"For writing: type of fake data to generate.");
declareProperty("CompressionFactor", 0.0, "The size of the file divided by the the size of the data on disk.",
Direction::Output);
declareProperty("SaveSpeed", 0.0, "The measured rate of saving the file, in MB (of data)/sec.", Direction::Output);
declareProperty("LoadSpeed", 0.0, "The measured rate of loading the file, in MB (of data)/sec.", Direction::Output);
}
//----------------------------------------------------------------------------------------------
/** Execute the algorithm.
*/
void NexusTester::exec() {
std::string SaveFilename = getPropertyValue("SaveFilename");
std::string LoadFilename = getPropertyValue("LoadFilename");
std::string FakeDataType = getPropertyValue("FakeData");
int ChunkSizeKb = getProperty("ChunkSize");
int NumChunks = getProperty("NumChunks");
bool Compress = getProperty("Compress");
if (ChunkSizeKb <= 0)
throw std::invalid_argument("ChunkSize must be > 0");
if (NumChunks <= 0)
throw std::invalid_argument("NumChunks must be > 0");
// Size of the chunk in number of integers
size_t chunkSize = ChunkSizeKb * 1024 / sizeof(uint32_t);
// ----------- Generate the fake data -----------------------------
auto fakeData = std::vector<uint32_t>(chunkSize);
if (FakeDataType == "Zeros") {
for (size_t i = 0; i < chunkSize; i++)
fakeData[i] = 0;
} else if (FakeDataType == "Incrementing Numbers") {
for (size_t i = 0; i < chunkSize; i++)
fakeData[i] = uint32_t(i);
} else if (FakeDataType == "Random Numbers") {
for (size_t i = 0; i < chunkSize; i++)
fakeData[i] = rand();
}
std::vector<int64_t> dims;
dims.emplace_back(int64_t(chunkSize) * NumChunks);
std::vector<int64_t> chunkDims;
chunkDims.emplace_back(int64_t(chunkSize));
// Total size in BYTES
double dataSizeMB = double(chunkSize * NumChunks * sizeof(uint32_t)) / (1024. * 1024.);
g_log.notice() << "Data size is " << dataSizeMB << " MB\n";
// ------------------------ Save a File ----------------------------
if (!SaveFilename.empty()) {
::NeXus::File file(SaveFilename, NXACC_CREATE5);
file.makeGroup("FakeDataGroup", "NXdata", true);
file.makeCompData("FakeData", ::NeXus::UINT32, dims, Compress ? ::NeXus::LZW : ::NeXus::NONE, chunkDims, true);
Progress prog(this, 0.0, 1.0, NumChunks);
CPUTimer tim;
for (int i = 0; i < NumChunks; i++) {
std::vector<int64_t> startDims;
startDims.emplace_back(i * int64_t(chunkSize));
file.putSlab(fakeData.data(), startDims, chunkDims);
prog.report();
}
file.close();
double seconds = tim.elapsedWallClock(false);
double MBperSec = dataSizeMB / seconds;
g_log.notice() << tim << " to save the file = " << MBperSec << " MB/sec\n";
this->setProperty("SaveSpeed", MBperSec);
}
// Check the size of the file created/loaded
Poco::File info(SaveFilename.empty() ? LoadFilename : SaveFilename);
double fileSizeMB = double(info.getSize()) / (1024. * 1024.);
g_log.notice() << "File size is " << fileSizeMB << " MB\n";
double CompressionFactor = fileSizeMB / dataSizeMB;
this->setProperty("CompressionFactor", CompressionFactor);
bool ClearDiskCache = this->getProperty("ClearDiskCache");
if (ClearDiskCache) {
g_log.information() << "Clearing disk cache.\n";
if (system("sync ; echo 3 > /proc/sys/vm/drop_caches") != 0)
g_log.error("Error clearing disk cache");
Poco::Thread::sleep(100);
}
// ------------------------ Load a File ----------------------------
if (!LoadFilename.empty()) {
::NeXus::File file(LoadFilename, NXACC_READ);
int HDFCacheSize = getProperty("HDFCacheSize");
NXsetcache(HDFCacheSize);
file.openGroup("FakeDataGroup", "NXdata");
Progress prog(this, 0.0, 1.0, NumChunks);
CPUTimer tim;
for (int i = 0; i < NumChunks; i++) {
file.openData("FakeData");
std::vector<int64_t> startDims;
startDims.emplace_back(i * int64_t(chunkSize));
file.getSlab(fakeData.data(), startDims, chunkDims);
prog.report();
file.closeData();
}
file.close();
double seconds = tim.elapsedWallClock(false);
double MBperSec = dataSizeMB / seconds;
this->setProperty("LoadSpeed", MBperSec);
g_log.notice() << tim << " to load the file = " << MBperSec << " MB/sec (data), " << MBperSec * CompressionFactor
<< " MB/sec (file)\n";
}
}
} // namespace DataHandling
} // namespace Mantid