-
Notifications
You must be signed in to change notification settings - Fork 2
/
io.cxx
216 lines (198 loc) · 5.46 KB
/
io.cxx
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
#include <asdf/io.hxx>
#include <asdf/asdf.hxx>
#include <asdf/ndarray.hxx>
#include <yaml-cpp/yaml.h>
#include <fstream>
namespace ASDF {
const string asdf_format_version = "1.0.0";
const string asdf_standard_version = "1.1.0";
bool have_checksum() {
#ifdef ASDF_HAVE_OPENSSL
return true;
#else
return false;
#endif
}
bool have_compression_blosc() {
#ifdef ASDF_HAVE_BLOSC
return true;
#else
return false;
#endif
}
bool have_compression_blosc2() {
#ifdef ASDF_HAVE_BLOSC2
return true;
#else
return false;
#endif
}
bool have_compression_bzip2() {
#ifdef ASDF_HAVE_BZIP2
return true;
#else
return false;
#endif
}
bool have_compression_liblz4() {
#ifdef ASDF_HAVE_LIBLZ4
return true;
#else
return false;
#endif
}
bool have_compression_libzstd() {
#ifdef ASDF_HAVE_LIBZSTD
return true;
#else
return false;
#endif
}
bool have_compression_zlib() {
#ifdef ASDF_HAVE_ZLIB
return true;
#else
return false;
#endif
}
// I/O
std::ostream &operator<<(std::ostream &os, block_format_t block_format) {
switch (block_format) {
case block_format_t::block:
return os << "block";
case block_format_t::inline_array:
return os << "inline_array";
default:
return os << "unknown";
}
}
std::ostream &operator<<(std::ostream &os, compression_t compression) {
switch (compression) {
case compression_t::none:
return os << "none";
case compression_t::blosc:
return os << "blosc";
case compression_t::blosc2:
return os << "blosc2";
case compression_t::bzip2:
return os << "bzip2";
case compression_t::liblz4:
return os << "liblz4";
case compression_t::libzstd:
return os << "libzstd";
case compression_t::zlib:
return os << "zlib";
default:
return os << "unknown";
}
}
reader_state::reader_state(const YAML::Node &tree,
const shared_ptr<istream> &pis,
const string &filename)
: filename(filename), tree(tree) {
for (;;) {
const auto [block, block_info] = ndarray::read_block(pis);
if (!block.valid())
break;
blocks.push_back(std::move(block));
block_infos.push_back(std::move(block_info));
}
}
block_info_t reader_state::get_block_info(int64_t index) const {
assert(index >= 0);
return block_infos.at(index);
}
YAML::Node reader_state::resolve_reference(const vector<string> &path) const {
// We allocate a new YAML node each time we take a step. If we don't
// do this, yaml-cpp will instead only create a reference (alias) to
// the new node, thus effectively overwriting the "tree" field.
auto node = unique_ptr<YAML::Node>(new YAML::Node(tree));
assert(node->IsDefined());
for (const auto &elem : path) {
if (node->IsSequence()) {
int idx = [=]() {
try {
return stoi(elem);
} catch (exception &) {
assert(0);
}
}();
node = unique_ptr<YAML::Node>(new YAML::Node((*node)[idx]));
} else if (node->IsMap()) {
node = unique_ptr<YAML::Node>(new YAML::Node((*node)[elem]));
} else {
// Could not resolve reference
// TODO: Output an actual error message
assert(0);
}
assert(node->IsDefined());
}
return *node;
}
pair<shared_ptr<reader_state>, YAML::Node>
reader_state::resolve_reference(const shared_ptr<reader_state> &rs,
const string &filename,
const vector<string> &path) {
shared_ptr<reader_state> refrs;
if (filename.empty()) {
// Read from same file
refrs = rs;
} else {
// Read from external file
string ref_filename;
if (!filename.empty() && filename[0] == '/') {
// absolute path
ref_filename = filename;
} else {
// preprend current path
assert(!rs->filename.empty()); // We could allow this
auto slashpos = rs->filename.rfind('/');
if (slashpos == string::npos)
ref_filename = filename;
else
ref_filename = rs->filename.substr(0, slashpos + 1) + filename;
}
if (!rs->other_files.count(ref_filename)) {
auto pis = make_shared<ifstream>(ref_filename, ios::binary | ios::in);
auto doc = asdf::from_yaml((istream &)*pis);
rs->other_files[ref_filename] =
make_shared<reader_state>(doc, pis, ref_filename);
}
refrs = rs->other_files.at(ref_filename);
}
auto node = refrs->resolve_reference(path);
return make_pair(refrs, node);
}
writer::writer(ostream &os, const map<string, string> &tags)
: os(os), emitter(os) {
// yaml-cpp does not support comments without leading space
os << "#ASDF " << asdf_format_version << "\n"
<< "#ASDF_STANDARD " << asdf_standard_version << "\n"
<< "# This is an ASDF file <https://asdf-standard.readthedocs.io/>\n"
// yaml-cpp does not support writing a YAML tag
<< "%YAML 1.1\n"
<< "%TAG ! tag:stsci.edu:asdf/\n";
for (const auto &kv : tags)
os << "%TAG !" << kv.first << "! " << kv.second << "\n";
emitter << YAML::BeginDoc;
}
writer::~writer() { assert(tasks.empty()); }
void writer::flush() {
emitter << YAML::EndDoc;
if (!tasks.empty()) {
YAML::Emitter index;
index << YAML::BeginDoc << YAML::Flow << YAML::BeginSeq;
for (auto &&task : tasks) {
index << os.tellp();
std::move(task)(os);
}
tasks.clear();
index << YAML::EndSeq << YAML::EndDoc;
// yaml-cpp does not support comments without leading space
os << "#ASDF BLOCK INDEX\n"
// yaml-cpp does not support writing a YAML tag
<< "%YAML 1.1\n"
<< index.c_str();
}
}
} // namespace ASDF