Skip to content
Permalink
Browse files
ARROW-4543: [C#] Update Flat Buffers code to latest version
- Update FlatBuffers code to latest version.
- Mark all FlatBuffer types as internal.

Note: I didn't use the latest `.fbs` file version because it included the SparseTensor support. Using the latest `.fbs` change caused problems with the `Tensor` generation. I have a [question if the latest `.fbs` file is correct](#2546 (comment)).

/cc @chutchinson @stephentoub - FYI

Author: Eric Erhardt <eric.erhardt@microsoft.com>

Closes #3637 from eerhardt/UpdateFlatBufferCode and squashes the following commits:

15e0831 <Eric Erhardt> Adding how to update FlatBuffers code to the README
043ce7e <Eric Erhardt> ARROW-4543:  Update Flat Buffers code to latest version
  • Loading branch information
eerhardt authored and kszucs committed Feb 18, 2019
1 parent c8007dd commit 9600d8b85d44af8d59f1f680341021717738a7b1
Showing with 565 additions and 180 deletions.
  1. +8 −0 csharp/README.md
  2. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Block.cs
  3. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Buffer.cs
  4. +1 −1 csharp/src/Apache.Arrow/Flatbuf/DictionaryBatch.cs
  5. +1 −1 csharp/src/Apache.Arrow/Flatbuf/DictionaryEncoding.cs
  6. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/DateUnit.cs
  7. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/Endianness.cs
  8. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/IntervalUnit.cs
  9. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/MessageHeader.cs
  10. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/MetadataVersion.cs
  11. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/Precision.cs
  12. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/TimeUnit.cs
  13. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/Type.cs
  14. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Enums/UnionMode.cs
  15. +8 −1 csharp/src/Apache.Arrow/Flatbuf/Field.cs
  16. +1 −1 csharp/src/Apache.Arrow/Flatbuf/FieldNode.cs
  17. +1 −1 csharp/src/Apache.Arrow/Flatbuf/FixedSizeBinary.cs
  18. +1 −1 csharp/src/Apache.Arrow/Flatbuf/FixedSizeList.cs
  19. +370 −123 csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBuffer.cs
  20. +1 −1 csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/ByteBufferUtil.cs
  21. +70 −2 csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferBuilder.cs
  22. +1 −1 csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/FlatBufferConstants.cs
  23. +1 −1 csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/IFlatbufferObject.cs
  24. +3 −3 csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Offset.cs
  25. +1 −1 csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Struct.cs
  26. +19 −1 csharp/src/Apache.Arrow/Flatbuf/FlatBuffers/Table.cs
  27. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Footer.cs
  28. +11 −1 csharp/src/Apache.Arrow/Flatbuf/KeyValue.cs
  29. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Map.cs
  30. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Message.cs
  31. +1 −1 csharp/src/Apache.Arrow/Flatbuf/RecordBatch.cs
  32. +3 −1 csharp/src/Apache.Arrow/Flatbuf/Schema.cs
  33. +8 −1 csharp/src/Apache.Arrow/Flatbuf/Tensor.cs
  34. +8 −1 csharp/src/Apache.Arrow/Flatbuf/TensorDim.cs
  35. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Binary.cs
  36. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Bool.cs
  37. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Date.cs
  38. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Decimal.cs
  39. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/FloatingPoint.cs
  40. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Int.cs
  41. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Interval.cs
  42. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/List.cs
  43. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Null.cs
  44. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Struct_.cs
  45. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Time.cs
  46. +6 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Timestamp.cs
  47. +7 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Union.cs
  48. +1 −1 csharp/src/Apache.Arrow/Flatbuf/Types/Utf8.cs
  49. +2 −2 csharp/src/Apache.Arrow/Ipc/ArrowStreamReader.cs
  50. +6 −6 csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
@@ -148,3 +148,11 @@ Build from the Apache Arrow project root.
dotnet test test/Apache.Arrow.Tests

All build artifacts are placed in the **artifacts** folder in the project root.

# Updating FlatBuffers code

See https://google.github.io/flatbuffers/flatbuffers_guide_use_java_c-sharp.html for how to get the `flatc` executable.

Run `flatc --csharp` on each `.fbs` file in the [format](../format) folder. And replace the checked in `.cs` files under [FlatBuf](src/Apache.Arrow/Flatbuf) with the generated files.

Update the non-generated [FlatBuffers](src/Apache.Arrow/Flatbuf/FlatBuffers) `.cs` files with the files from the [google/flatbuffers repo](https://github.com/google/flatbuffers/tree/master/net/FlatBuffers).
@@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf
using global::System;
using global::FlatBuffers;

public struct Block : IFlatbufferObject
internal struct Block : IFlatbufferObject
{
private Struct __p;
public ByteBuffer ByteBuffer { get { return __p.bb; } }
@@ -10,7 +10,7 @@ namespace Apache.Arrow.Flatbuf

/// ----------------------------------------------------------------------
/// A Buffer represents a single contiguous memory segment
public struct Buffer : IFlatbufferObject
internal struct Buffer : IFlatbufferObject
{
private Struct __p;
public ByteBuffer ByteBuffer { get { return __p.bb; } }
@@ -14,7 +14,7 @@ namespace Apache.Arrow.Flatbuf
/// There is one vector / column per dictionary, but that vector / column
/// may be spread across multiple dictionary batches by using the isDelta
/// flag
public struct DictionaryBatch : IFlatbufferObject
internal struct DictionaryBatch : IFlatbufferObject
{
private Table __p;
public ByteBuffer ByteBuffer { get { return __p.bb; } }
@@ -10,7 +10,7 @@ namespace Apache.Arrow.Flatbuf

/// ----------------------------------------------------------------------
/// Dictionary encoding metadata
public struct DictionaryEncoding : IFlatbufferObject
internal struct DictionaryEncoding : IFlatbufferObject
{
private Table __p;
public ByteBuffer ByteBuffer { get { return __p.bb; } }
@@ -5,7 +5,7 @@
namespace Apache.Arrow.Flatbuf
{

public enum DateUnit : short
internal enum DateUnit : short
{
DAY = 0,
MILLISECOND = 1,
@@ -7,7 +7,7 @@ namespace Apache.Arrow.Flatbuf

/// ----------------------------------------------------------------------
/// Endianness of the platform producing the data
public enum Endianness : short
internal enum Endianness : short
{
Little = 0,
Big = 1,
@@ -5,7 +5,7 @@
namespace Apache.Arrow.Flatbuf
{

public enum IntervalUnit : short
internal enum IntervalUnit : short
{
YEAR_MONTH = 0,
DAY_TIME = 1,
@@ -13,7 +13,7 @@ namespace Apache.Arrow.Flatbuf
/// Arrow implementations do not need to implement all of the message types,
/// which may include experimental metadata types. For maximum compatibility,
/// it is best to send data using RecordBatch
public enum MessageHeader : byte
internal enum MessageHeader : byte
{
NONE = 0,
Schema = 1,
@@ -5,7 +5,7 @@
namespace Apache.Arrow.Flatbuf
{

public enum MetadataVersion : short
internal enum MetadataVersion : short
{
/// 0.1.0
V1 = 0,
@@ -5,7 +5,7 @@
namespace Apache.Arrow.Flatbuf
{

public enum Precision : short
internal enum Precision : short
{
HALF = 0,
SINGLE = 1,
@@ -5,7 +5,7 @@
namespace Apache.Arrow.Flatbuf
{

public enum TimeUnit : short
internal enum TimeUnit : short
{
SECOND = 0,
MILLISECOND = 1,
@@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf
/// ----------------------------------------------------------------------
/// Top-level Type value, enabling extensible type-specific metadata. We can
/// add new logical types to Type without breaking backwards compatibility
public enum Type : byte
internal enum Type : byte
{
NONE = 0,
Null = 1,
@@ -5,7 +5,7 @@
namespace Apache.Arrow.Flatbuf
{

public enum UnionMode : short
internal enum UnionMode : short
{
Sparse = 0,
Dense = 1,
@@ -15,7 +15,7 @@ namespace Apache.Arrow.Flatbuf
/// - children is only for nested Arrow arrays
/// - For primitive types, children will have length 0
/// - nullable should default to true in general
public struct Field : IFlatbufferObject
internal struct Field : IFlatbufferObject
{
private Table __p;
public ByteBuffer ByteBuffer { get { return __p.bb; } }
@@ -25,7 +25,12 @@ public struct Field : IFlatbufferObject
public Field __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; }

public string Name { get { int o = __p.__offset(4); return o != 0 ? __p.__string(o + __p.bb_pos) : null; } }
#if ENABLE_SPAN_T
public Span<byte> GetNameBytes() { return __p.__vector_as_span(4); }
#else
public ArraySegment<byte>? GetNameBytes() { return __p.__vector_as_arraysegment(4); }
#endif
public byte[] GetNameArray() { return __p.__vector_as_array<byte>(4); }
public bool Nullable { get { int o = __p.__offset(6); return o != 0 ? 0!=__p.bb.Get(o + __p.bb_pos) : (bool)false; } }
public Type TypeType { get { int o = __p.__offset(8); return o != 0 ? (Type)__p.bb.Get(o + __p.bb_pos) : Flatbuf.Type.NONE; } }
public TTable? Type<TTable>() where TTable : struct, IFlatbufferObject { int o = __p.__offset(10); return o != 0 ? (TTable?)__p.__union<TTable>(o) : null; }
@@ -62,9 +67,11 @@ public struct Field : IFlatbufferObject
public static void AddDictionary(FlatBufferBuilder builder, Offset<DictionaryEncoding> dictionaryOffset) { builder.AddOffset(4, dictionaryOffset.Value, 0); }
public static void AddChildren(FlatBufferBuilder builder, VectorOffset childrenOffset) { builder.AddOffset(5, childrenOffset.Value, 0); }
public static VectorOffset CreateChildrenVector(FlatBufferBuilder builder, Offset<Field>[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); }
public static VectorOffset CreateChildrenVectorBlock(FlatBufferBuilder builder, Offset<Field>[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); }
public static void StartChildrenVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); }
public static void AddCustomMetadata(FlatBufferBuilder builder, VectorOffset customMetadataOffset) { builder.AddOffset(6, customMetadataOffset.Value, 0); }
public static VectorOffset CreateCustomMetadataVector(FlatBufferBuilder builder, Offset<KeyValue>[] data) { builder.StartVector(4, data.Length, 4); for (int i = data.Length - 1; i >= 0; i--) builder.AddOffset(data[i].Value); return builder.EndVector(); }
public static VectorOffset CreateCustomMetadataVectorBlock(FlatBufferBuilder builder, Offset<KeyValue>[] data) { builder.StartVector(4, data.Length, 4); builder.Add(data); return builder.EndVector(); }
public static void StartCustomMetadataVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(4, numElems, 4); }
public static Offset<Field> EndField(FlatBufferBuilder builder) {
int o = builder.EndObject();
@@ -17,7 +17,7 @@ namespace Apache.Arrow.Flatbuf
/// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
/// would have {length: 5, null_count: 2} for its List node, and {length: 6,
/// null_count: 0} for its Int16 node, as separate FieldNode structs
public struct FieldNode : IFlatbufferObject
internal struct FieldNode : IFlatbufferObject
{
private Struct __p;
public ByteBuffer ByteBuffer { get { return __p.bb; } }
@@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf
using global::System;
using global::FlatBuffers;

public struct FixedSizeBinary : IFlatbufferObject
internal struct FixedSizeBinary : IFlatbufferObject
{
private Table __p;
public ByteBuffer ByteBuffer { get { return __p.bb; } }
@@ -8,7 +8,7 @@ namespace Apache.Arrow.Flatbuf
using global::System;
using global::FlatBuffers;

public struct FixedSizeList : IFlatbufferObject
internal struct FixedSizeList : IFlatbufferObject
{
private Table __p;
public ByteBuffer ByteBuffer { get { return __p.bb; } }

0 comments on commit 9600d8b

Please sign in to comment.