Skip to content

Optimise Blaze JSON output for space-efficiency #505

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jviotti merged 1 commit into from
Sep 4, 2025
Merged

Optimise Blaze JSON output for space-efficiency #505

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 71 additions & 40 deletions src/compiler/compile_json.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,57 +1,88 @@
#include <sourcemeta/blaze/compiler.h>

#include <cassert> // assert
#include <variant> // std::visit
#include <cassert> // assert
#include <string_view> // std::string_view
#include <variant> // std::visit

namespace {
auto to_json(const sourcemeta::blaze::Instruction &instruction)
auto to_json(const sourcemeta::blaze::Instruction &instruction,
std::vector<sourcemeta::core::JSON::String> &resources)
-> sourcemeta::core::JSON {
auto result{sourcemeta::core::JSON::make_object()};
// Note that we purposely avoid objects to help consumers avoid potentially
// expensive hash-map or flat-map lookups when parsing back
auto result{sourcemeta::core::JSON::make_array()};

// We use single characters to save space, as this serialised format
// is not meant to be human-readable anyway
result.assign("t", sourcemeta::core::to_json(instruction.type));
result.assign(
"s", sourcemeta::core::to_json(instruction.relative_schema_location));
result.assign(
"i", sourcemeta::core::to_json(instruction.relative_instance_location));
result.assign("k", sourcemeta::core::to_json(instruction.keyword_location));
result.assign("r", sourcemeta::core::to_json(instruction.schema_resource));

auto value{sourcemeta::core::JSON::make_object()};
value.assign("t", sourcemeta::core::to_json(instruction.value.index()));
value.assign("v", std::visit(
[](const auto &variant) {
return sourcemeta::core::to_json(variant);
},
instruction.value));
result.assign("v", std::move(value));

assert(result.at("v").is_object());
assert(result.at("v").size() == 2);
assert(result.at("v").defines("t"));
assert(result.at("v").defines("v"));
assert(result.at("v").at("t").is_integer());

auto children_json{sourcemeta::core::JSON::make_array()};
result.assign("c", sourcemeta::core::to_json(instruction.children,
[](const auto &subinstruction) {
return to_json(subinstruction);
}));
result.push_back(sourcemeta::core::to_json(instruction.type));

result.push_back(
sourcemeta::core::to_json(instruction.relative_schema_location));
result.push_back(
sourcemeta::core::to_json(instruction.relative_instance_location));

const auto match{instruction.keyword_location.find('#')};
if (instruction.schema_resource > 0 && match != std::string::npos) {
if (resources.size() < instruction.schema_resource) {
resources.resize(instruction.schema_resource);
}

if (resources[instruction.schema_resource - 1].empty()) {
resources[instruction.schema_resource - 1] =
instruction.keyword_location.substr(0, match);
}

result.push_back(
sourcemeta::core::JSON{instruction.keyword_location.substr(match)});
} else {
result.push_back(sourcemeta::core::to_json(instruction.keyword_location));
}

result.push_back(sourcemeta::core::to_json(instruction.schema_resource));

// Note that we purposely avoid objects to help consumers avoid potentially
// expensive hash-map or flat-map lookups when parsing back
auto value{sourcemeta::core::JSON::make_array()};
const auto value_index{instruction.value.index()};
value.push_back(sourcemeta::core::to_json(value_index));
// Don't encode empty values, which tend to happen a lot
if (value_index != 0) {
value.push_back(std::visit(
[](const auto &variant) { return sourcemeta::core::to_json(variant); },
instruction.value));
}
assert(value.is_array());
assert(!value.empty());
assert(value.at(0).is_integer());
result.push_back(std::move(value));

if (!instruction.children.empty()) {
auto children_json{sourcemeta::core::JSON::make_array()};
result.push_back(sourcemeta::core::to_json(
instruction.children, [&resources](const auto &subinstruction) {
return to_json(subinstruction, resources);
}));
}

return result;
}
} // namespace

namespace sourcemeta::blaze {

auto to_json(const Template &schema_template) -> sourcemeta::core::JSON {
auto result{sourcemeta::core::JSON::make_object()};
result.assign("dynamic", sourcemeta::core::JSON{schema_template.dynamic});
result.assign("track", sourcemeta::core::JSON{schema_template.track});
result.assign("instructions",
sourcemeta::core::to_json(schema_template.instructions,
[](const auto &instruction) {
return ::to_json(instruction);
}));
// Note that we purposely avoid objects to help consumers avoid potentially
// expensive hash-map or flat-map lookups when parsing back
auto result{sourcemeta::core::JSON::make_array()};
result.push_back(sourcemeta::core::JSON{schema_template.dynamic});
result.push_back(sourcemeta::core::JSON{schema_template.track});
std::vector<sourcemeta::core::JSON::String> resources;
auto instructions{sourcemeta::core::to_json(
schema_template.instructions, [&resources](const auto &instruction) {
return ::to_json(instruction, resources);
})};
result.push_back(sourcemeta::core::to_json(resources));
result.push_back(std::move(instructions));
return result;
}

Expand Down
155 changes: 82 additions & 73 deletions src/evaluator/evaluator_json.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,49 +5,49 @@
namespace {
auto value_from_json(const sourcemeta::core::JSON &wrapper)
-> std::optional<sourcemeta::blaze::Value> {
if (!wrapper.is_object()) {
if (!wrapper.is_array() || wrapper.array_size() == 0 ||
!wrapper.at(0).is_integer()) {
return std::nullopt;
} else if (wrapper.array_size() == 1) {
return sourcemeta::blaze::ValueNone{};
}

const auto type{wrapper.try_at("t")};
const auto value{wrapper.try_at("v")};
if (!type || !value) {
return std::nullopt;
}
const auto &value{wrapper.at(1)};

using namespace sourcemeta::blaze;
switch (type->to_integer()) {
switch (wrapper.at(0).to_integer()) {
// clang-format off
case 0: return sourcemeta::core::from_json<ValueNone>(*value);
case 1: return sourcemeta::core::from_json<ValueJSON>(*value);
case 2: return sourcemeta::core::from_json<ValueSet>(*value);
case 3: return sourcemeta::core::from_json<ValueString>(*value);
case 4: return sourcemeta::core::from_json<ValueProperty>(*value);
case 5: return sourcemeta::core::from_json<ValueStrings>(*value);
case 6: return sourcemeta::core::from_json<ValueStringSet>(*value);
case 7: return sourcemeta::core::from_json<ValueTypes>(*value);
case 8: return sourcemeta::core::from_json<ValueType>(*value);
case 9: return sourcemeta::core::from_json<ValueRegex>(*value);
case 10: return sourcemeta::core::from_json<ValueUnsignedInteger>(*value);
case 11: return sourcemeta::core::from_json<ValueRange>(*value);
case 12: return sourcemeta::core::from_json<ValueBoolean>(*value);
case 13: return sourcemeta::core::from_json<ValueNamedIndexes>(*value);
case 14: return sourcemeta::core::from_json<ValueStringType>(*value);
case 15: return sourcemeta::core::from_json<ValueStringMap>(*value);
case 16: return sourcemeta::core::from_json<ValuePropertyFilter>(*value);
case 17: return sourcemeta::core::from_json<ValueIndexPair>(*value);
case 18: return sourcemeta::core::from_json<ValuePointer>(*value);
case 19: return sourcemeta::core::from_json<ValueTypedProperties>(*value);
case 20: return sourcemeta::core::from_json<ValueStringHashes>(*value);
case 21: return sourcemeta::core::from_json<ValueTypedHashes>(*value);
case 0: return ValueNone{};
case 1: return sourcemeta::core::from_json<ValueJSON>(value);
case 2: return sourcemeta::core::from_json<ValueSet>(value);
case 3: return sourcemeta::core::from_json<ValueString>(value);
case 4: return sourcemeta::core::from_json<ValueProperty>(value);
case 5: return sourcemeta::core::from_json<ValueStrings>(value);
case 6: return sourcemeta::core::from_json<ValueStringSet>(value);
case 7: return sourcemeta::core::from_json<ValueTypes>(value);
case 8: return sourcemeta::core::from_json<ValueType>(value);
case 9: return sourcemeta::core::from_json<ValueRegex>(value);
case 10: return sourcemeta::core::from_json<ValueUnsignedInteger>(value);
case 11: return sourcemeta::core::from_json<ValueRange>(value);
case 12: return sourcemeta::core::from_json<ValueBoolean>(value);
case 13: return sourcemeta::core::from_json<ValueNamedIndexes>(value);
case 14: return sourcemeta::core::from_json<ValueStringType>(value);
case 15: return sourcemeta::core::from_json<ValueStringMap>(value);
case 16: return sourcemeta::core::from_json<ValuePropertyFilter>(value);
case 17: return sourcemeta::core::from_json<ValueIndexPair>(value);
case 18: return sourcemeta::core::from_json<ValuePointer>(value);
case 19: return sourcemeta::core::from_json<ValueTypedProperties>(value);
case 20: return sourcemeta::core::from_json<ValueStringHashes>(value);
case 21: return sourcemeta::core::from_json<ValueTypedHashes>(value);
// clang-format on
default:
assert(false);
return ValueNone{};
}
}

auto instructions_from_json(const sourcemeta::core::JSON &instructions)
auto instructions_from_json(const sourcemeta::core::JSON &instructions,
const sourcemeta::core::JSON &resources)
-> std::optional<sourcemeta::blaze::Instructions> {
if (!instructions.is_array()) {
return std::nullopt;
Expand All @@ -56,42 +56,36 @@ auto instructions_from_json(const sourcemeta::core::JSON &instructions)
sourcemeta::blaze::Instructions result;
result.reserve(instructions.size());
for (const auto &instruction : instructions.as_array()) {
if (!instruction.is_object()) {
if (!instruction.is_array() || instruction.array_size() < 6) {
return std::nullopt;
}

const auto type{instruction.try_at("t")};
const auto relative_schema_location{instruction.try_at("s")};
const auto relative_instance_location{instruction.try_at("i")};
const auto keyword_location{instruction.try_at("k")};
const auto schema_resource{instruction.try_at("r")};
const auto value{instruction.try_at("v")};
const auto children{instruction.try_at("c")};

if (!type || !relative_schema_location || !relative_instance_location ||
!keyword_location || !schema_resource || !value || !children ||
!type->is_positive() || !relative_schema_location->is_string() ||
!relative_instance_location->is_string() ||
!keyword_location->is_string() || !schema_resource->is_positive() ||
!value->is_object() || !children->is_array()) {
return std::nullopt;
}
const auto &type{instruction.at(0)};
const auto &relative_schema_location{instruction.at(1)};
const auto &relative_instance_location{instruction.at(2)};
const auto &keyword_location{instruction.at(3)};
const auto &schema_resource{instruction.at(4)};
const auto &value{instruction.at(5)};

auto type_result{
sourcemeta::core::from_json<sourcemeta::blaze::InstructionIndex>(
*type)};
sourcemeta::core::from_json<sourcemeta::blaze::InstructionIndex>(type)};
auto relative_schema_location_result{
sourcemeta::core::from_json<sourcemeta::core::Pointer>(
*relative_schema_location)};
relative_schema_location)};
auto relative_instance_location_result{
sourcemeta::core::from_json<sourcemeta::core::Pointer>(
*relative_instance_location)};
relative_instance_location)};
auto keyword_location_result{
sourcemeta::core::from_json<std::string>(*keyword_location)};
sourcemeta::core::from_json<std::string>(keyword_location)};
auto schema_resource_result{
sourcemeta::core::from_json<std::size_t>(*schema_resource)};
auto value_result{value_from_json(*value)};
auto children_result{instructions_from_json(*children)};
sourcemeta::core::from_json<std::size_t>(schema_resource)};
auto value_result{value_from_json(value)};

// Parse children if there
std::optional<sourcemeta::blaze::Instructions> children_result{
instruction.array_size() == 7
? instructions_from_json(instruction.at(6), resources)
: sourcemeta::blaze::Instructions{}};

if (!type_result.has_value() ||
!relative_schema_location_result.has_value() ||
Expand All @@ -102,14 +96,28 @@ auto instructions_from_json(const sourcemeta::core::JSON &instructions)
return std::nullopt;
}

// TODO: Maybe we should emplace here?
result.push_back({std::move(type_result).value(),
std::move(relative_schema_location_result).value(),
std::move(relative_instance_location_result).value(),
std::move(keyword_location_result).value(),
std::move(schema_resource_result).value(),
std::move(value_result).value(),
std::move(children_result).value()});
if (schema_resource_result.value() > 0 &&
resources.array_size() >= schema_resource_result.value() &&
keyword_location_result.value().starts_with('#')) {
// TODO: Maybe we should emplace here?
result.push_back(
{std::move(type_result).value(),
std::move(relative_schema_location_result).value(),
std::move(relative_instance_location_result).value(),
resources.at(schema_resource_result.value() - 1).to_string() +
std::move(keyword_location_result).value(),
schema_resource_result.value(), std::move(value_result).value(),
std::move(children_result).value()});
} else {
// TODO: Maybe we should emplace here?
result.push_back({std::move(type_result).value(),
std::move(relative_schema_location_result).value(),
std::move(relative_instance_location_result).value(),
std::move(keyword_location_result).value(),
std::move(schema_resource_result).value(),
std::move(value_result).value(),
std::move(children_result).value()});
}
}

return result;
Expand All @@ -120,26 +128,27 @@ auto instructions_from_json(const sourcemeta::core::JSON &instructions)
namespace sourcemeta::blaze {

auto from_json(const sourcemeta::core::JSON &json) -> std::optional<Template> {
if (!json.is_object()) {
if (!json.is_array() || json.array_size() != 4) {
return std::nullopt;
}

const auto instructions{json.try_at("instructions")};
const auto dynamic{json.try_at("dynamic")};
const auto track{json.try_at("track")};
if (!instructions || !dynamic || !track) {
const auto &dynamic{json.at(0)};
const auto &track{json.at(1)};
const auto &resources{json.at(2)};

if (!dynamic.is_boolean() || !track.is_boolean() || !resources.is_array()) {
return std::nullopt;
}

auto instructions_result{instructions_from_json(*instructions)};
if (!instructions_result.has_value() || !dynamic->is_boolean() ||
!track->is_boolean()) {
const auto &instructions{json.at(3)};
auto instructions_result{instructions_from_json(instructions, resources)};
if (!instructions_result.has_value()) {
return std::nullopt;
}

return Template{.instructions = std::move(instructions_result).value(),
.dynamic = dynamic->to_boolean(),
.track = track->to_boolean()};
.dynamic = dynamic.to_boolean(),
.track = track.to_boolean()};
}

} // namespace sourcemeta::blaze
Loading
Loading