serialize.c.erb

#include "yarp.h"

#include <stdio.h>

static inline uint32_t
yp_ptrdifft_to_u32(ptrdiff_t value) {
    assert(value >= 0 && ((unsigned long) value) < UINT32_MAX);
    return (uint32_t) value;
}

static inline uint32_t
yp_sizet_to_u32(size_t value) {
    assert(value < UINT32_MAX);
    return (uint32_t) value;
}

static void
yp_serialize_location(yp_parser_t *parser, yp_location_t *location, yp_buffer_t *buffer) {
    assert(location->start);
    assert(location->end);
    assert(location->start <= location->end);

    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(location->start - parser->start));
    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(location->end - location->start));
}

static void
yp_serialize_string(yp_parser_t *parser, yp_string_t *string, yp_buffer_t *buffer) {
    switch (string->type) {
        case YP_STRING_SHARED: {
            yp_buffer_append_u8(buffer, 1);
            yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(yp_string_source(string) - parser->start));
            yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_string_length(string)));
            break;
        }
        case YP_STRING_OWNED:
        case YP_STRING_CONSTANT: {
            uint32_t length = yp_sizet_to_u32(yp_string_length(string));
            yp_buffer_append_u8(buffer, 2);
            yp_buffer_append_u32(buffer, length);
            yp_buffer_append_bytes(buffer, yp_string_source(string), length);
            break;
        }
        case YP_STRING_MAPPED:
            assert(false && "Cannot serialize mapped strings.");
            break;
    }
}

void
yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
    yp_buffer_append_u8(buffer, (uint8_t) YP_NODE_TYPE(node));

    size_t offset = buffer->length;

    yp_serialize_location(parser, &node->location, buffer);

    switch (YP_NODE_TYPE(node)) {
        // We do not need to serialize a ScopeNode ever as
        // it is not part of the AST
        case YP_SCOPE_NODE:
            return;
        <%- nodes.each do |node| -%>
        case <%= node.type %>: {
            <%- if node.needs_serialized_length? -%>
            // serialize length
            // encoding of location u32s make us need to save this offset.
            size_t length_offset = buffer->length;
            yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
            <%- end -%>
            <%- node.fields.each do |field| -%>
            <%- case field -%>
            <%- when YARP::NodeField -%>
            yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
            <%- when YARP::OptionalNodeField -%>
            if (((yp_<%= node.human %>_t *)node)-><%= field.name %> == NULL) {
                yp_buffer_append_u8(buffer, 0);
            } else {
                yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
            }
            <%- when YARP::StringField -%>
            yp_serialize_string(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
            <%- when YARP::NodeListField -%>
            uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size);
            yp_buffer_append_u32(buffer, <%= field.name %>_size);
            for (uint32_t index = 0; index < <%= field.name %>_size; index++) {
                yp_serialize_node(parser, (yp_node_t *) ((yp_<%= node.human %>_t *)node)-><%= field.name %>.nodes[index], buffer);
            }
            <%- when YARP::ConstantField -%>
            yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>));
            <%- when YARP::ConstantListField -%>
            uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size);
            yp_buffer_append_u32(buffer, <%= field.name %>_size);
            for (uint32_t index = 0; index < <%= field.name %>_size; index++) {
                yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]));
            }
            <%- when YARP::LocationField -%>
            yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
            <%- when YARP::OptionalLocationField -%>
            if (((yp_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) {
                yp_buffer_append_u8(buffer, 0);
            } else {
                yp_buffer_append_u8(buffer, 1);
                yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
            }
            <%- when YARP::UInt32Field -%>
            yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= field.name %>);
            <%- when YARP::FlagsField -%>
            yp_buffer_append_u32(buffer, node->flags >> <%= YARP::COMMON_FLAGS %>);
            <%- else -%>
            <%- raise -%>
            <%- end -%>
            <%- end -%>
            <%- if node.needs_serialized_length? -%>
            // serialize length
            uint32_t length = yp_sizet_to_u32(buffer->length - offset - sizeof(uint32_t));
            memcpy(buffer->value + length_offset, &length, sizeof(uint32_t));
            <%- end -%>
            break;
        }
        <%- end -%>
    }
}

static void
yp_serialize_comment(yp_parser_t *parser, yp_comment_t *comment, yp_buffer_t *buffer) {
    // serialize type
    yp_buffer_append_u8(buffer, (uint8_t) comment->type);

    // serialize location
    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->start - parser->start));
    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->end - comment->start));
}

static void
yp_serialize_comment_list(yp_parser_t *parser, yp_list_t *list, yp_buffer_t *buffer) {
    yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(list)));

    yp_comment_t *comment;
    for (comment = (yp_comment_t *) list->head; comment != NULL; comment = (yp_comment_t *) comment->node.next) {
        yp_serialize_comment(parser, comment, buffer);
    }
}

static void
yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, yp_buffer_t *buffer) {
    // serialize message
    size_t message_length = strlen(diagnostic->message);
    yp_buffer_append_u32(buffer, yp_sizet_to_u32(message_length));
    yp_buffer_append_str(buffer, diagnostic->message, message_length);

    // serialize location
    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(diagnostic->start - parser->start));
    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(diagnostic->end - diagnostic->start));
}

static void
yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t *list, yp_buffer_t *buffer) {
    yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(list)));

    yp_diagnostic_t *diagnostic;
    for (diagnostic = (yp_diagnostic_t *) list->head; diagnostic != NULL; diagnostic = (yp_diagnostic_t *) diagnostic->node.next) {
        yp_serialize_diagnostic(parser, diagnostic, buffer);
    }
}

static void
yp_serialize_encoding(yp_encoding_t *encoding, yp_buffer_t *buffer) {
    size_t encoding_length = strlen(encoding->name);
    yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length));
    yp_buffer_append_str(buffer, encoding->name, encoding_length);
}

#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
void
yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
    yp_serialize_encoding(&parser->encoding, buffer);
    yp_serialize_comment_list(parser, &parser->comment_list, buffer);
    yp_serialize_diagnostic_list(parser, &parser->error_list, buffer);
    yp_serialize_diagnostic_list(parser, &parser->warning_list, buffer);

    // Here we're going to leave space for the offset of the constant pool in
    // the buffer.
    size_t offset = buffer->length;
    yp_buffer_append_zeroes(buffer, 4);

    // Next, encode the length of the constant pool.
    yp_buffer_append_u32(buffer, yp_sizet_to_u32(parser->constant_pool.size));

    // Now we're going to serialize the content of the node.
    yp_serialize_node(parser, node, buffer);

    // Now we're going to serialize the offset of the constant pool back where
    // we left space for it.
    uint32_t length = yp_sizet_to_u32(buffer->length);
    memcpy(buffer->value + offset, &length, sizeof(uint32_t));

    // Now we're going to serialize the constant pool.
    offset = buffer->length;
    yp_buffer_append_zeroes(buffer, parser->constant_pool.size * 8);

    yp_constant_t *constant;
    for (size_t index = 0; index < parser->constant_pool.capacity; index++) {
        constant = &parser->constant_pool.constants[index];

        // If we find a constant at this index, serialize it at the correct
        // index in the buffer.
        if (constant->id != 0) {
            size_t buffer_offset = offset + ((constant->id - 1) * 8);

            uint32_t source_offset = yp_ptrdifft_to_u32(constant->start - parser->start);
            uint32_t constant_length = yp_sizet_to_u32(constant->length);

            memcpy(buffer->value + buffer_offset, &source_offset, 4);
            memcpy(buffer->value + buffer_offset + 4, &constant_length, 4);
        }
    }
}

static void
serialize_token(void *data, yp_parser_t *parser, yp_token_t *token) {
    yp_buffer_t *buffer = (yp_buffer_t *) data;

    yp_buffer_append_u32(buffer, token->type);
    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->start - parser->start));
    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->end - token->start));
    yp_buffer_append_u32(buffer, parser->lex_state);
}

YP_EXPORTED_FUNCTION void
yp_lex_serialize(const uint8_t *source, size_t size, const char *filepath, yp_buffer_t *buffer) {
    yp_parser_t parser;
    yp_parser_init(&parser, source, size, filepath);

    yp_lex_callback_t lex_callback = (yp_lex_callback_t) {
        .data = (void *) buffer,
        .callback = serialize_token,
    };

    parser.lex_callback = &lex_callback;
    yp_node_t *node = yp_parse(&parser);

    // Append 0 to mark end of tokens
    yp_buffer_append_u8(buffer, 0);

    yp_serialize_encoding(&parser.encoding, buffer);
    yp_serialize_comment_list(&parser, &parser.comment_list, buffer);
    yp_serialize_diagnostic_list(&parser, &parser.error_list, buffer);
    yp_serialize_diagnostic_list(&parser, &parser.warning_list, buffer);

    yp_node_destroy(&parser, node);
    yp_parser_free(&parser);
}

// Parse and serialize both the AST and the tokens represented by the given
// source to the given buffer.
YP_EXPORTED_FUNCTION void
yp_parse_lex_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata) {
    yp_parser_t parser;
    yp_parser_init(&parser, source, size, NULL);
    if (metadata) yp_parser_metadata(&parser, metadata);

    yp_lex_callback_t lex_callback = (yp_lex_callback_t) {
        .data = (void *) buffer,
        .callback = serialize_token,
    };

    parser.lex_callback = &lex_callback;
    yp_node_t *node = yp_parse(&parser);

    yp_buffer_append_u8(buffer, 0);
    yp_serialize(&parser, node, buffer);

    yp_node_destroy(&parser, node);
    yp_parser_free(&parser);
}