Skip to content

Commit

Permalink
ARROW-8178: [C++] Update to Flatbuffers 1.12.0
Browse files Browse the repository at this point in the history 
This includes some UBSAN-related fixes that make our `MakeNonNull` helper function unneeded with Flatbuffers

Closes #6678 from wesm/ARROW-8178

Authored-by: Wes McKinney <wesm+git@apache.org>
Signed-off-by: Wes McKinney <wesm+git@apache.org>
  • Loading branch information
@wesm
wesm committed Mar 20, 2020
1 parent 4f9db53 commit 54b87c8
Show file tree
Hide file tree
Showing 12 changed files with 1,465 additions and 1,018 deletions.
23 changes: 9 additions & 14 deletions cpp/src/arrow/ipc/metadata_internal.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -677,8 +677,7 @@ class FieldToFlatbufferVisitor {
type_ids.push_back(code);
}

auto fb_type_ids =
fbb_.CreateVector(util::MakeNonNull(type_ids.data()), type_ids.size());
auto fb_type_ids = fbb_.CreateVector(type_ids.data(), type_ids.size());

type_offset_ = flatbuf::CreateUnion(fbb_, mode, fb_type_ids).Union();
return Status::OK();
Expand All @@ -701,8 +700,7 @@ class FieldToFlatbufferVisitor {
Status GetResult(const std::shared_ptr<Field>& field, FieldOffset* offset) {
auto fb_name = fbb_.CreateString(field->name());
RETURN_NOT_OK(VisitType(*field->type()));
auto fb_children =
fbb_.CreateVector(util::MakeNonNull(children_.data()), children_.size());
auto fb_children = fbb_.CreateVector(children_.data(), children_.size());

DictionaryOffset dictionary = 0;
if (field->type()->id() == Type::DICTIONARY) {
Expand Down Expand Up @@ -855,7 +853,7 @@ static Status WriteFieldNodes(FBB& fbb, const std::vector<FieldMetadata>& nodes,
}
fb_nodes.emplace_back(node.length, node.null_count);
}
*out = fbb.CreateVectorOfStructs(util::MakeNonNull(fb_nodes.data()), fb_nodes.size());
*out = fbb.CreateVectorOfStructs(fb_nodes.data(), fb_nodes.size());
return Status::OK();
}

Expand All @@ -868,8 +866,7 @@ static Status WriteBuffers(FBB& fbb, const std::vector<BufferMetadata>& buffers,
const BufferMetadata& buffer = buffers[i];
fb_buffers.emplace_back(buffer.offset, buffer.length);
}
*out =
fbb.CreateVectorOfStructs(util::MakeNonNull(fb_buffers.data()), fb_buffers.size());
*out = fbb.CreateVectorOfStructs(fb_buffers.data(), fb_buffers.size());

return Status::OK();
}
Expand Down Expand Up @@ -900,9 +897,8 @@ Status MakeSparseTensorIndexCOO(FBB& fbb, const SparseCOOIndex& sparse_index,
auto indices_type_offset =
flatbuf::CreateInt(fbb, index_value_type.bit_width(), index_value_type.is_signed());

auto fb_strides =
fbb.CreateVector(util::MakeNonNull(sparse_index.indices()->strides().data()),
sparse_index.indices()->strides().size());
auto fb_strides = fbb.CreateVector(sparse_index.indices()->strides().data(),
sparse_index.indices()->strides().size());

const BufferMetadata& indices_metadata = buffers[0];
flatbuf::Buffer indices(indices_metadata.offset, indices_metadata.length);
Expand Down Expand Up @@ -1072,11 +1068,10 @@ Result<std::shared_ptr<Buffer>> WriteTensorMessage(const Tensor& tensor,
dims.push_back(flatbuf::CreateTensorDim(fbb, tensor.shape()[i], name));
}

auto fb_shape = fbb.CreateVector(util::MakeNonNull(dims.data()), dims.size());
auto fb_shape = fbb.CreateVector(dims.data(), dims.size());

flatbuffers::Offset<flatbuffers::Vector<int64_t>> fb_strides;
fb_strides = fbb.CreateVector(util::MakeNonNull(tensor.strides().data()),
tensor.strides().size());
fb_strides = fbb.CreateVector(tensor.strides().data(), tensor.strides().size());
int64_t body_length = tensor.size() * elem_size;
flatbuf::Buffer buffer(buffer_start_offset, body_length);

Expand Down Expand Up @@ -1119,7 +1114,7 @@ FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
}

return fbb.CreateVectorOfStructs(util::MakeNonNull(fb_blocks.data()), fb_blocks.size());
return fbb.CreateVectorOfStructs(fb_blocks.data(), fb_blocks.size());
}

Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
Expand Down
59 changes: 31 additions & 28 deletions cpp/src/generated/File_generated.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ namespace arrow {
namespace flatbuf {

struct Footer;
struct FooterBuilder;

struct Block;

Expand Down Expand Up @@ -55,28 +56,29 @@ FLATBUFFERS_STRUCT_END(Block, 24);
/// Arrow File metadata
///
struct Footer FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef FooterBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_VERSION = 4,
VT_SCHEMA = 6,
VT_DICTIONARIES = 8,
VT_RECORDBATCHES = 10,
VT_CUSTOM_METADATA = 12
};
MetadataVersion version() const {
return static_cast<MetadataVersion>(GetField<int16_t>(VT_VERSION, 0));
org::apache::arrow::flatbuf::MetadataVersion version() const {
return static_cast<org::apache::arrow::flatbuf::MetadataVersion>(GetField<int16_t>(VT_VERSION, 0));
}
const Schema *schema() const {
return GetPointer<const Schema *>(VT_SCHEMA);
const org::apache::arrow::flatbuf::Schema *schema() const {
return GetPointer<const org::apache::arrow::flatbuf::Schema *>(VT_SCHEMA);
}
const flatbuffers::Vector<const Block *> *dictionaries() const {
return GetPointer<const flatbuffers::Vector<const Block *> *>(VT_DICTIONARIES);
const flatbuffers::Vector<const org::apache::arrow::flatbuf::Block *> *dictionaries() const {
return GetPointer<const flatbuffers::Vector<const org::apache::arrow::flatbuf::Block *> *>(VT_DICTIONARIES);
}
const flatbuffers::Vector<const Block *> *recordBatches() const {
return GetPointer<const flatbuffers::Vector<const Block *> *>(VT_RECORDBATCHES);
const flatbuffers::Vector<const org::apache::arrow::flatbuf::Block *> *recordBatches() const {
return GetPointer<const flatbuffers::Vector<const org::apache::arrow::flatbuf::Block *> *>(VT_RECORDBATCHES);
}
/// User-defined metadata
const flatbuffers::Vector<flatbuffers::Offset<KeyValue>> *custom_metadata() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<KeyValue>> *>(VT_CUSTOM_METADATA);
const flatbuffers::Vector<flatbuffers::Offset<org::apache::arrow::flatbuf::KeyValue>> *custom_metadata() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<org::apache::arrow::flatbuf::KeyValue>> *>(VT_CUSTOM_METADATA);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
Expand All @@ -95,21 +97,22 @@ struct Footer FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
};

struct FooterBuilder {
typedef Footer Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_version(MetadataVersion version) {
void add_version(org::apache::arrow::flatbuf::MetadataVersion version) {
fbb_.AddElement<int16_t>(Footer::VT_VERSION, static_cast<int16_t>(version), 0);
}
void add_schema(flatbuffers::Offset<Schema> schema) {
void add_schema(flatbuffers::Offset<org::apache::arrow::flatbuf::Schema> schema) {
fbb_.AddOffset(Footer::VT_SCHEMA, schema);
}
void add_dictionaries(flatbuffers::Offset<flatbuffers::Vector<const Block *>> dictionaries) {
void add_dictionaries(flatbuffers::Offset<flatbuffers::Vector<const org::apache::arrow::flatbuf::Block *>> dictionaries) {
fbb_.AddOffset(Footer::VT_DICTIONARIES, dictionaries);
}
void add_recordBatches(flatbuffers::Offset<flatbuffers::Vector<const Block *>> recordBatches) {
void add_recordBatches(flatbuffers::Offset<flatbuffers::Vector<const org::apache::arrow::flatbuf::Block *>> recordBatches) {
fbb_.AddOffset(Footer::VT_RECORDBATCHES, recordBatches);
}
void add_custom_metadata(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<KeyValue>>> custom_metadata) {
void add_custom_metadata(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<org::apache::arrow::flatbuf::KeyValue>>> custom_metadata) {
fbb_.AddOffset(Footer::VT_CUSTOM_METADATA, custom_metadata);
}
explicit FooterBuilder(flatbuffers::FlatBufferBuilder &_fbb)
Expand All @@ -126,11 +129,11 @@ struct FooterBuilder {

inline flatbuffers::Offset<Footer> CreateFooter(
flatbuffers::FlatBufferBuilder &_fbb,
MetadataVersion version = MetadataVersion::V1,
flatbuffers::Offset<Schema> schema = 0,
flatbuffers::Offset<flatbuffers::Vector<const Block *>> dictionaries = 0,
flatbuffers::Offset<flatbuffers::Vector<const Block *>> recordBatches = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<KeyValue>>> custom_metadata = 0) {
org::apache::arrow::flatbuf::MetadataVersion version = org::apache::arrow::flatbuf::MetadataVersion::V1,
flatbuffers::Offset<org::apache::arrow::flatbuf::Schema> schema = 0,
flatbuffers::Offset<flatbuffers::Vector<const org::apache::arrow::flatbuf::Block *>> dictionaries = 0,
flatbuffers::Offset<flatbuffers::Vector<const org::apache::arrow::flatbuf::Block *>> recordBatches = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<org::apache::arrow::flatbuf::KeyValue>>> custom_metadata = 0) {
FooterBuilder builder_(_fbb);
builder_.add_custom_metadata(custom_metadata);
builder_.add_recordBatches(recordBatches);
Expand All @@ -142,14 +145,14 @@ inline flatbuffers::Offset<Footer> CreateFooter(

inline flatbuffers::Offset<Footer> CreateFooterDirect(
flatbuffers::FlatBufferBuilder &_fbb,
MetadataVersion version = MetadataVersion::V1,
flatbuffers::Offset<Schema> schema = 0,
const std::vector<Block> *dictionaries = nullptr,
const std::vector<Block> *recordBatches = nullptr,
const std::vector<flatbuffers::Offset<KeyValue>> *custom_metadata = nullptr) {
auto dictionaries__ = dictionaries ? _fbb.CreateVectorOfStructs<Block>(*dictionaries) : 0;
auto recordBatches__ = recordBatches ? _fbb.CreateVectorOfStructs<Block>(*recordBatches) : 0;
auto custom_metadata__ = custom_metadata ? _fbb.CreateVector<flatbuffers::Offset<KeyValue>>(*custom_metadata) : 0;
org::apache::arrow::flatbuf::MetadataVersion version = org::apache::arrow::flatbuf::MetadataVersion::V1,
flatbuffers::Offset<org::apache::arrow::flatbuf::Schema> schema = 0,
const std::vector<org::apache::arrow::flatbuf::Block> *dictionaries = nullptr,
const std::vector<org::apache::arrow::flatbuf::Block> *recordBatches = nullptr,
const std::vector<flatbuffers::Offset<org::apache::arrow::flatbuf::KeyValue>> *custom_metadata = nullptr) {
auto dictionaries__ = dictionaries ? _fbb.CreateVectorOfStructs<org::apache::arrow::flatbuf::Block>(*dictionaries) : 0;
auto recordBatches__ = recordBatches ? _fbb.CreateVectorOfStructs<org::apache::arrow::flatbuf::Block>(*recordBatches) : 0;
auto custom_metadata__ = custom_metadata ? _fbb.CreateVector<flatbuffers::Offset<org::apache::arrow::flatbuf::KeyValue>>(*custom_metadata) : 0;
return org::apache::arrow::flatbuf::CreateFooter(
_fbb,
version,
Expand Down
Loading

0 comments on commit 54b87c8

Please sign in to comment.