Skip to content

Commit

Permalink
corrade-rc,Utility: implement null termination and alignment.
Browse files Browse the repository at this point in the history 
Both done in the same bulk of work as internally it's the same feature
-- just arbitrary padding between successive data.

Apologies for not having alignment working properly for overriden groups
-- it'd need to have dynamic allocator support (i.e., passing an
allocator to an arbitrary API) first, and hooking them up with Path,
which is a significant amount of work. For most cases it's fine already
like this, as the default alignment is 2*sizeof(void*) usually, and
people usually need just 4 or 8 anyway.
  • Loading branch information
@mosra
mosra committed Jan 1, 2023
1 parent ecb7b41 commit 9ef4b4a
Show file tree
Hide file tree
Showing 26 changed files with 961 additions and 68 deletions.
2 changes: 2 additions & 0 deletions doc/corrade-changelog.dox
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ namespace Corrade {
- New @ref Corrade/Utility/Math.h header implementing @ref Utility::min() and
@ref Utility::max() because having to @cpp #include <algorithm> @ce to get
@ref std::min() and @ref std::max() is unacceptable.
- @ref Utility::Resource and @ref corrade-rc "corrade-rc" is now capable of
optional making resources null-terminated and arbitrarily aligned
- Added @ref Utility::String::lowercaseInPlace() and @relativeref{Utility::String,uppercaseInPlace()}
together with @ref Utility::String::lowercase() and
@relativeref{Utility::String,uppercase()} overloads taking a
Expand Down
21 changes: 14 additions & 7 deletions src/Corrade/Utility/Implementation/Resource.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,23 @@
namespace Corrade { namespace Utility { namespace Implementation {

inline Containers::StringView resourceFilenameAt(const unsigned int* const positions, const unsigned char* const filenames, const std::size_t i) {
/* Every position pair denotes end offsets of one file, filename is first */
const std::size_t begin = i == 0 ? 0 : positions[2*(i - 1)];
const std::size_t end = positions[2*i];
/* Every position pair denotes end offsets of one file, filename is first
and the upper 8 bits are reserved for padding */
const std::size_t begin = (i == 0 ? 0 : positions[2*(i - 1)]) & 0x00ffffffu;
const std::size_t end = (positions[2*i]) & 0x00ffffffu;
return {reinterpret_cast<const char*>(filenames) + begin, end - begin, Containers::StringViewFlag::Global};
}

inline Containers::StringView resourceDataAt(const unsigned int* const positions, const unsigned char* const data, const std::size_t i) {
/* Every position pair denotes end offsets of one file, data is second */
const std::size_t begin = i == 0 ? 0 : positions[2*(i - 1) + 1];
const std::size_t end = positions[2*i + 1];
return {reinterpret_cast<const char*>(data) + begin, end - begin, Containers::StringViewFlag::Global};
/* If there's any padding after (contained in the upper 8 bits of
filename), the data can be marked as null-terminated. This can be either
deliberate (a single null byte added after) or "accidental" due to for
example padding for alignment. */
const std::size_t padding = (positions[2*i + 0] >> 24 & 0xff);
return {reinterpret_cast<const char*>(data) + begin, end - begin - padding, Containers::StringViewFlag::Global|(padding ? Containers::StringViewFlag::NullTerminated : Containers::StringViewFlag{})};
}

/* Assuming the filenames are sorted, look up a particular filename. Returns
Expand All @@ -52,14 +58,15 @@ inline std::size_t resourceLookup(const unsigned int count, const unsigned int*
/* Like std::map, but without crazy allocations using std::lower_bound and
a std::lexicographical_compare */
struct Position {
unsigned int filename;
unsigned int filenamePadding;
unsigned int data;
};
auto positions = Containers::arrayCast<const Position>(Containers::arrayView(positionData, count*2));
const Position* found = std::lower_bound(positions.begin(), positions.end(), filename,
[positions, filenames](const Position& position, const Containers::StringView filename) {
const std::size_t end = position.filename;
const std::size_t begin = &position == positions ? 0 : (&position - 1)->filename;
/* The upper 8 bits of filename are reserved for padding */
const std::size_t end = position.filenamePadding & 0x00ffffffu;
const std::size_t begin = &position == positions ? 0 : (&position - 1)->filenamePadding & 0x00ffffffu;
/* Not constructing a temporary StringView here as this shall be
faster */
/** @todo Actually, temporary StringView *could* be faster because
Expand Down
142 changes: 116 additions & 26 deletions src/Corrade/Utility/Implementation/ResourceCompile.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,21 +55,31 @@ namespace Corrade { namespace Utility { namespace Implementation { namespace {

/** @todo this whole thing needs a serious cleanup and deSTLification */

std::string hexcode(const Containers::ArrayView<const char> data) {
std::string hexcode(const Containers::ArrayView<const char> data, std::size_t padding = 0) {
std::ostringstream out;
out << std::hex;

const std::size_t dataSizeWithPadding = data.size() + padding;

/* Each row is indented by four spaces and has newline at the end */
for(std::size_t row = 0; row < data.size(); row += 15) {
for(std::size_t row = 0; row < dataSizeWithPadding; row += 15) {
out << " ";

/* Convert all characters on a row to hex "0xab,0x01,..." */
for(std::size_t end = Utility::min(row + 15, data.size()), i = row; i != end; ++i) {
const std::size_t end = Utility::min(row + 15, data.size());
for(std::size_t i = row; i < end; ++i) {
out << "0x" << std::setw(2) << std::setfill('0')
<< static_cast<unsigned int>(static_cast<unsigned char>(data[i]))
<< ",";
}

/* Padding bytes after the actual data, if any. Printing 0 instead of
0x00 for easier distinction between data and padding. */
const std::size_t paddingEnd = Utility::min(row + 15, dataSizeWithPadding);
for(std::size_t i = Utility::max(end, row); i < paddingEnd; ++i) {
out << " 0,";
}

out << "\n";
}

Expand All @@ -78,6 +88,8 @@ std::string hexcode(const Containers::ArrayView<const char> data) {

struct FileData {
Containers::StringView filename;
bool nullTerminated;
unsigned int align;
Containers::Array<char> data;
};

Expand All @@ -89,6 +101,16 @@ inline bool lessFilename(const FileData& a, const FileData& b) {
CORRADE_RESOURCE_INITIALIZE(), group name is the one to load the resources
from. Output is a C++ file with hexadecimal data representation. */
Containers::String resourceCompile(const Containers::StringView name, const Containers::StringView group, const Containers::ArrayView<const FileData> files) {
using namespace Containers::Literals;

/* We're sorting by filename in order to have efficient lookup, which may
not be as memory-efficient when alignment is involved. A more
memory-efficient way to pack the data would be by sorting by alignment,
but that only works if the data size is actually divisible by its
alignment, which is true in C but not in general. Plus it would go
against the filename sorting, meaning each filename would need to store
offset + size and not just offset, which means extra overhead even if
nothing actually needs the alignment. */
CORRADE_ASSERT(std::is_sorted(files.begin(), files.end(), lessFilename),
"Utility::Resource::compile(): the file list is not sorted", {});

Expand Down Expand Up @@ -125,31 +147,68 @@ int resourceFinalizer_{0}() {{
)", name, group);
}

unsigned int maxAlign = 1;
for(const FileData& file: files) {
CORRADE_INTERNAL_ASSERT(file.align && file.align <= 128 && !(file.align & (file.align - 1)));
maxAlign = Utility::max(maxAlign, file.align);
}

std::string positions, filenames, data;
unsigned int filenamesLen = 0, dataLen = 0;
unsigned int filenamesLen = 0, dataLen = 0, minDataLen = 0;

/* Convert data to hexacodes */
for(const FileData& file: files) {
for(std::size_t i = 0; i != files.size(); ++i) {
const FileData& file = files[i];

filenamesLen += file.filename.size();
/* The filenames shouldn't span more than 16 MB, because then it would
run into the 8 bits reserved for padding */
CORRADE_INTERNAL_ASSERT(!(filenamesLen & 0xff000000u));

/* Minimal data length to satisfy alignment -- for a non-empty file
aligned to N bytes there has to be at least N bytes of data, even if
the file is shorter than that */
if(file.data.size())
minDataLen = Utility::max(minDataLen, dataLen + file.align);

dataLen += file.data.size();

Utility::formatInto(positions, positions.size(), " 0x{:.8x},0x{:.8x},\n", filenamesLen, dataLen);
/* Next file offset before alignment. Add a 1-byte padding if this file
is meant to be null-terminated. */
const unsigned int nextOffset = dataLen + (file.nullTerminated ? 1 : 0);

/* Next file offset. If this is the last file, take into account the
minimal data length given by alignment of any previous files.
Otherwise align the next file according to its alignment. */
unsigned int nextOffsetAligned;
if(i == files.size() - 1) {
nextOffsetAligned = Utility::max(nextOffset, minDataLen);
} else {
const FileData& nextFile = files[i + 1];
nextOffsetAligned = nextFile.align*((nextOffset + nextFile.align - 1)/nextFile.align);
}

const unsigned int padding = nextOffsetAligned - dataLen;
dataLen = nextOffsetAligned;

CORRADE_INTERNAL_ASSERT(padding < 256);
Utility::formatInto(positions, positions.size(), " 0x{:.8x},0x{:.8x},\n", filenamesLen | (padding << 24), dataLen);

Utility::formatInto(filenames, filenames.size(), "\n /* {} */\n", file.filename);
filenames += hexcode(Containers::StringView{file.filename});

Utility::formatInto(data, data.size(), "\n /* {} */\n", file.filename);
data += hexcode(file.data);
data += hexcode(file.data, padding);
}

/* Remove last comma and newline from the positions and filenames array */
positions.resize(positions.size() - 2);
filenames.resize(filenames.size() - 2);

/* Remove last newline from the data array, remove also the preceding comma
if the last file is not empty */
if present (from either data or alignment) */
data.resize(data.size() - 1);
if(!files.back().data.isEmpty())
if(Containers::StringView{data}.hasSuffix(','))
data.resize(data.size() - 1);

/* Return C++ file. The functions have forward declarations to avoid warning
Expand All @@ -164,45 +223,50 @@ int resourceFinalizer_{0}() {{
namespace {{
/* Pair `i` is offset of filename `i + 1` in the low 24 bits, padding after
data `i` in the upper 8 bits, and a 32bit offset of data `i + 1`. Offset of
the first filename and data is implicitly 0. */
const unsigned int resourcePositions[] = {{
{0}
}};
const unsigned char resourceFilenames[] = {{{1}
}};
{2}const unsigned char resourceData[] = {{{3}
{2}{3}const unsigned char resourceData[] = {{{4}
{2}}};
Corrade::Utility::Implementation::ResourceGroup resource;
}}
int resourceInitializer_{4}();
int resourceInitializer_{4}() {{
resource.name = "{5}";
resource.count = {6};
int resourceInitializer_{5}();
int resourceInitializer_{5}() {{
resource.name = "{6}";
resource.count = {7};
resource.positions = resourcePositions;
resource.filenames = resourceFilenames;
resource.data = {7};
resource.data = {8};
Corrade::Utility::Resource::registerData(resource);
return 1;
}} CORRADE_AUTOMATIC_INITIALIZER(resourceInitializer_{4})
}} CORRADE_AUTOMATIC_INITIALIZER(resourceInitializer_{5})
int resourceFinalizer_{4}();
int resourceFinalizer_{4}() {{
int resourceFinalizer_{5}();
int resourceFinalizer_{5}() {{
Corrade::Utility::Resource::unregisterData(resource);
return 1;
}} CORRADE_AUTOMATIC_FINALIZER(resourceFinalizer_{4})
}} CORRADE_AUTOMATIC_FINALIZER(resourceFinalizer_{5})
)",
positions, // 0
filenames, // 1
dataLen ? "" : "// ", // 2
data, // 3
name, // 4
group, // 5
files.size(), // 6
dataLen ? "resourceData" : "nullptr" // 7
dataLen ? ""_s : "// "_s, // 2
maxAlign == 1 ? Containers::String{} : // 3
Utility::format("alignas({}) ", maxAlign),
data, // 4
name, // 5
group, // 6
files.size(), // 7
dataLen ? "resourceData" : "nullptr" // 8
);
}

Expand All @@ -223,6 +287,20 @@ Containers::String resourceCompileFrom(const Containers::StringView name, const
}
const Containers::StringView group = conf.value<Containers::StringView>("group");

/* Global null termination and alignment options, false and 1 if not
present. Limited to 128 and not 256 in order to have the padding fit
into a byte -- if a null-terminated file is exactly 256 bytes, the
padding would need to be 256 again to have the next file 256-bit
aligned, which needs 9 bits. */
const bool globalNullTerminated = conf.value<bool>("nullTerminated");
/** @todo findValue(), once it exists */
const unsigned int globalAlign = conf.hasValue("align") ?
conf.value<unsigned int>("align") : 1;
if(!globalAlign || globalAlign > 128 || globalAlign & (globalAlign - 1)) {
Error{} << " Error: alignment in group" << group << "required to be a power-of-two value between 1 and 128, got" << globalAlign;
return {};
}

/* Load all files */
std::vector<const ConfigurationGroup*> files = conf.groups("file");
Containers::Array<FileData> fileData;
Expand All @@ -235,12 +313,24 @@ Containers::String resourceCompileFrom(const Containers::StringView name, const
return {};
}

/* Local null termination / alignment options. Falls back to the global
ones if not present. Limiting to 128 due to the reason above. */
/** @todo findValue(), once it exists */
const bool nullTerminated = file->hasValue("nullTerminated") ?
file->value<bool>("nullTerminated") : globalNullTerminated;
const unsigned int align = file->hasValue("align") ?
file->value<unsigned int>("align") : globalAlign;
if(!align || align > 128 || align & (align- 1)) {
Error{} << " Error: alignment of file" << fileData.size() + 1 << "in group" << group << "required to be a power-of-two value between 1 and 128, got" << align;
return {};
}

Containers::Optional<Containers::Array<char>> contents = Path::read(Path::join(path, filename));
if(!contents) {
Error() << " Error: cannot open file" << filename << "of file" << fileData.size()+1 << "in group" << group;
return {};
}
arrayAppend(fileData, InPlaceInit, alias, *std::move(contents));
arrayAppend(fileData, InPlaceInit, alias, nullTerminated, align, *std::move(contents));
}

/* The list has to be sorted before passing it to compile() */
Expand Down
Loading

0 comments on commit 9ef4b4a

Please sign in to comment.