Skip to content

Commit

Permalink
New class ptr_blob_t
Browse files Browse the repository at this point in the history
  • Loading branch information
@eschnett
eschnett committed May 25, 2018
1 parent e1db935 commit 743330f
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 28 deletions.
1 change: 1 addition & 0 deletions asdf.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ asdf::asdf(const reader_state &rs, const YAML::Node &node) {
// } else if (key == "table") {
// tab = make_shared<table>(rs, node["table"]);
} else {
#warning "TODO: check tag"
data[key] = make_shared<ndarray>(rs, kv.second);
}
}
Expand Down
2 changes: 2 additions & 0 deletions asdf_io.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class writer {
YAML::Emitter emitter;

// Tasks that write the blocks
// TODO: rename this variable
vector<function<void(ostream &os)>> tasks;

public:
Expand All @@ -68,6 +69,7 @@ class writer {
return w;
}

// TODO: rename this function
int64_t add_task(function<void(ostream &)> &&task) {
tasks.push_back(move(task));
return tasks.size() - 1;
Expand Down
49 changes: 35 additions & 14 deletions asdf_ndarray.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ class generic_blob_t {

virtual const void *ptr() const = 0;
virtual void *ptr() = 0;
virtual size_t bytes() const = 0;
virtual void reserve(size_t bytes) = 0;
virtual void resize(size_t bytes) = 0;
virtual size_t nbytes() const = 0;
virtual void reserve(size_t nbytes) = 0;
virtual void resize(size_t nbytes) = 0;
};

template <typename T> class blob_t : public generic_blob_t {
Expand All @@ -41,14 +41,14 @@ template <typename T> class blob_t : public generic_blob_t {

virtual const void *ptr() const { return data.data(); }
virtual void *ptr() { return data.data(); }
virtual size_t bytes() const { return data.size() * sizeof(T); }
virtual void reserve(size_t bytes) {
assert(bytes % sizeof(T) == 0);
data.reserve(bytes / sizeof(T));
virtual size_t nbytes() const { return data.size() * sizeof(T); }
virtual void reserve(size_t nbytes) {
assert(nbytes % sizeof(T) == 0);
data.reserve(nbytes / sizeof(T));
}
virtual void resize(size_t bytes) {
assert(bytes % sizeof(T) == 0);
data.resize(bytes / sizeof(T));
virtual void resize(size_t nbytes) {
assert(nbytes % sizeof(T) == 0);
data.resize(nbytes / sizeof(T));
}
};

Expand All @@ -66,9 +66,30 @@ template <> class blob_t<bool> : public generic_blob_t {

virtual const void *ptr() const { return data.data(); }
virtual void *ptr() { return data.data(); }
virtual size_t bytes() const { return data.size(); }
virtual void reserve(size_t bytes) { data.resize(bytes); }
virtual void resize(size_t bytes) { data.resize(bytes); }
virtual size_t nbytes() const { return data.size(); }
virtual void reserve(size_t nbytes) { data.resize(nbytes); }
virtual void resize(size_t nbytes) { data.resize(nbytes); }
};

class ptr_blob_t : public generic_blob_t {
void *data;
size_t size;

public:
ptr_blob_t() = delete;

ptr_blob_t(void *data, size_t size) : data(data), size(size) { assert(data); }
template <typename T>
ptr_blob_t(const vector<T> &data)
: data(data.data()), size(data.size() * sizeof(T)) {}

virtual ~ptr_blob_t() {}

virtual const void *ptr() const { return data; }
virtual void *ptr() { return data; }
virtual size_t nbytes() const { return size; }
virtual void reserve(size_t nbytes) { assert(0); }
virtual void resize(size_t nbytes) { assert(0); }
};

shared_ptr<generic_blob_t> read_block(istream &is);
Expand Down Expand Up @@ -109,7 +130,7 @@ class ndarray {
int64_t npoints = 1;
for (int d = 0; d < rank; ++d)
npoints *= shape[d];
assert(data->bytes() == npoints * datatype->type_size());
assert(data->nbytes() == npoints * datatype->type_size());
// Check mask
if (!mask.empty())
assert(mask.size() == npoints);
Expand Down
28 changes: 14 additions & 14 deletions ndarray.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ void ndarray::write_block(ostream &os) const {
comp = {'b', 'z', 'p', '2'};
// Allocate 600 bytes plus 1% more
outdata = make_shared<blob_t<unsigned char>>(vector<unsigned char>(
600 + data->bytes() + (data->bytes() + 99) / 100));
600 + data->nbytes() + (data->nbytes() + 99) / 100));
const int level = 9;
bz_stream strm;
strm.bzalloc = NULL;
Expand All @@ -330,8 +330,8 @@ void ndarray::write_block(ostream &os) const {
BZ2_bzCompressInit(&strm, level, 0, 0);
strm.next_in = reinterpret_cast<char *>(const_cast<void *>(data->ptr()));
strm.next_out = reinterpret_cast<char *>(outdata->ptr());
uint64_t avail_in = data->bytes();
uint64_t avail_out = outdata->bytes();
uint64_t avail_in = data->nbytes();
uint64_t avail_out = outdata->nbytes();
for (;;) {
uint64_t this_avail_in =
min(uint64_t(numeric_limits<unsigned int>::max()), avail_in);
Expand All @@ -348,8 +348,8 @@ void ndarray::write_block(ostream &os) const {
assert(iret == BZ_RUN_OK);
}
assert(avail_in == 0);
outdata->resize(outdata->bytes() - avail_out);
if (outdata->bytes() >= data->bytes()) {
outdata->resize(outdata->nbytes() - avail_out);
if (outdata->nbytes() >= data->nbytes()) {
// Skip compression if it does not reduce the size
comp = {0, 0, 0, 0};
outdata = data;
Expand All @@ -366,7 +366,7 @@ void ndarray::write_block(ostream &os) const {
comp = {'z', 'l', 'i', 'b'};
// Allocate 6 bytes plus 5 bytes per 16 kByte more
outdata = make_shared<blob_t<unsigned char>>(vector<unsigned char>(
(6 + data->bytes() + (data->bytes() + 16383) / 16384 * 5)));
(6 + data->nbytes() + (data->nbytes() + 16383) / 16384 * 5)));
const int level = 9;
z_stream strm;
strm.zalloc = Z_NULL;
Expand All @@ -377,8 +377,8 @@ void ndarray::write_block(ostream &os) const {
strm.next_in =
reinterpret_cast<unsigned char *>(const_cast<void *>(data->ptr()));
strm.next_out = reinterpret_cast<unsigned char *>(outdata->ptr());
uint64_t avail_in = data->bytes();
uint64_t avail_out = outdata->bytes();
uint64_t avail_in = data->nbytes();
uint64_t avail_out = outdata->nbytes();
for (;;) {
uint64_t this_avail_in =
min(uint64_t(numeric_limits<uInt>::max()), avail_in);
Expand All @@ -395,8 +395,8 @@ void ndarray::write_block(ostream &os) const {
assert(iret == Z_OK);
}
assert(avail_in == 0);
outdata->resize(outdata->bytes() - avail_out);
if (outdata->bytes() >= data->bytes()) {
outdata->resize(outdata->nbytes() - avail_out);
if (outdata->nbytes() >= data->nbytes()) {
// Skip compression if it does not reduce the size
comp = {0, 0, 0, 0};
outdata = data;
Expand All @@ -414,20 +414,20 @@ void ndarray::write_block(ostream &os) const {
for (auto ch : comp)
output(header, ch);
// allocated_space
uint64_t allocated_space = outdata->bytes();
uint64_t allocated_space = outdata->nbytes();
output(header, allocated_space);
// used_space
uint64_t used_space = allocated_space; // no padding
output(header, used_space);
// data_space
uint64_t data_space = data->bytes();
uint64_t data_space = data->nbytes();
output(header, data_space);
// checksum
array<unsigned char, 16> checksum;
#ifdef HAVE_OPENSSL
MD5_CTX ctx;
MD5_Init(&ctx);
MD5_Update(&ctx, outdata->ptr(), outdata->bytes());
MD5_Update(&ctx, outdata->ptr(), outdata->nbytes());
MD5_Final(checksum.data(), &ctx);
#else
checksum = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
Expand All @@ -443,7 +443,7 @@ void ndarray::write_block(ostream &os) const {
// write header
os.write(reinterpret_cast<const char *>(header.data()), header.size());
// write data
os.write(reinterpret_cast<const char *>(outdata->ptr()), outdata->bytes());
os.write(reinterpret_cast<const char *>(outdata->ptr()), outdata->nbytes());
// write padding
vector<char> padding(allocated_space - used_space);
os.write(padding.data(), padding.size());
Expand Down

0 comments on commit 743330f

Please sign in to comment.