-
Notifications
You must be signed in to change notification settings - Fork 22
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add Flatbuffer Schema Support #70
Changes from 9 commits
15c29ff
5f2e676
058467e
d345d27
d3ac125
ff4b563
e274c21
5e38b8b
c251616
5e11a5c
45fc71a
3909e08
6104a53
3a6a8a5
fa625ec
141de2e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,129 @@ | ||
import { generateFlatbuffers } from "./generateFlatbuffers"; | ||
import { exampleEnum, exampleMessage } from "./testFixtures"; | ||
|
||
describe("generateFlatBuffer", () => { | ||
it("generates Message .fbs files", () => { | ||
expect(generateFlatbuffers(exampleMessage)).toMatchInlineSnapshot(` | ||
"// Generated by https://github.com/foxglove/schemas | ||
|
||
include "ByteVector.fbs"; | ||
include "Duration.fbs"; | ||
include "ExampleEnum.fbs"; | ||
include "NestedMessage.fbs"; | ||
include "Time.fbs"; | ||
|
||
namespace foxglove; | ||
|
||
/// An example type | ||
table ExampleMessage { | ||
/// duration field | ||
field_duration:Duration = 1; | ||
|
||
/// time field | ||
field_time:Time = 1; | ||
|
||
/// boolean field | ||
field_boolean:bool = true; | ||
|
||
/// bytes field | ||
field_bytes:[uint8] = 1; | ||
|
||
/// float64 field | ||
field_float64:double = 1; | ||
|
||
/// uint32 field | ||
field_uint32:uint32 = 1; | ||
|
||
/// string field | ||
field_string:string = 1; | ||
|
||
/// duration array field | ||
field_duration_array:[Duration]; | ||
|
||
/// time array field | ||
field_time_array:[Time]; | ||
|
||
/// boolean array field | ||
field_boolean_array:[bool]; | ||
|
||
/// bytes array field | ||
field_bytes_array:[ByteVector]; | ||
|
||
/// float64 array field | ||
field_float64_array:[double]; | ||
|
||
/// uint32 array field | ||
field_uint32_array:[uint32]; | ||
|
||
/// string array field | ||
field_string_array:[string]; | ||
|
||
/// duration fixed-length array field | ||
/// length 3 | ||
field_duration_fixed_array:[Duration]; | ||
|
||
/// time fixed-length array field | ||
/// length 3 | ||
field_time_fixed_array:[Time]; | ||
|
||
/// boolean fixed-length array field | ||
/// length 3 | ||
field_boolean_fixed_array:[bool]; | ||
|
||
/// bytes fixed-length array field | ||
/// length 3 | ||
field_bytes_fixed_array:[ByteVector]; | ||
|
||
/// float64 fixed-length array field | ||
/// length 3 | ||
field_float64_fixed_array:[double]; | ||
|
||
/// uint32 fixed-length array field | ||
/// length 3 | ||
field_uint32_fixed_array:[uint32]; | ||
|
||
/// string fixed-length array field | ||
/// length 3 | ||
field_string_fixed_array:[string]; | ||
|
||
/// An enum field | ||
field_enum:ExampleEnum; | ||
|
||
/// An enum array field | ||
field_enum_array:[ExampleEnum]; | ||
|
||
/// A nested field | ||
field_nested:foxglove.NestedMessage; | ||
|
||
/// A nested array field | ||
/// With | ||
/// a | ||
/// very | ||
/// long | ||
/// description | ||
field_nested_array:[foxglove.NestedMessage]; | ||
} | ||
|
||
root_type ExampleMessage; | ||
" | ||
`); | ||
}); | ||
it("generates Enum .fbs files", () => { | ||
expect(generateFlatbuffers(exampleEnum)).toMatchInlineSnapshot(` | ||
"// Generated by https://github.com/foxglove/schemas | ||
|
||
namespace foxglove; | ||
|
||
/// An example enum | ||
enum ExampleEnum : ubyte { | ||
/// Value A | ||
A = 1, | ||
|
||
/// Value B | ||
B = 2, | ||
} | ||
|
||
" | ||
`); | ||
}); | ||
}); |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,146 @@ | ||
import { FoxglovePrimitive, FoxgloveSchema } from "./types"; | ||
|
||
// Flatbuffers only supports nested vectors via table | ||
export const BYTE_VECTOR_FB = ` | ||
namespace foxglove; | ||
|
||
/// Used for nesting byte vectors | ||
table ByteVector { | ||
data:[uint8]; | ||
} | ||
root_type ByteVector; | ||
`; | ||
|
||
// Same as protobuf wellknown types | ||
export const TIME_FB = ` | ||
namespace foxglove; | ||
|
||
struct Time { | ||
/// Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z | ||
sec:uint64; | ||
/// Nano-second fractions from 0 to 999,999,999 inclusive | ||
nsec:uint32; | ||
} | ||
`; | ||
|
||
export const DURATION_FB = ` | ||
namespace foxglove; | ||
|
||
struct Duration { | ||
/// Signed seconds of the span of time. Must be from -315,576,000,000 to +315,576,000,000 inclusive. | ||
sec:int64; | ||
/// if sec === 0 : -999,999,999 <= nsec <= +999,999,999 | ||
/// otherwise sign of sec must match sign of nsec or be 0 and abs(nsec) <= 999,999,999 | ||
nsec:int32; | ||
} | ||
`; | ||
|
||
function primitiveToFlatbuffers(type: Exclude<FoxglovePrimitive, "time" | "duration">) { | ||
switch (type) { | ||
case "uint32": | ||
return "uint32"; | ||
case "bytes": | ||
return "[uint8]"; | ||
case "string": | ||
return "string"; | ||
case "boolean": | ||
return "bool"; | ||
case "float64": | ||
return "double"; | ||
} | ||
} | ||
|
||
export function generateFlatbuffers(schema: FoxgloveSchema): string { | ||
let definition; | ||
const imports = new Set<string>(); | ||
switch (schema.type) { | ||
case "enum": { | ||
const fields = schema.values.map(({ name, value, description }) => { | ||
if (description != undefined) { | ||
return `/// ${description}\n ${name} = ${value},`; | ||
} else { | ||
return `${name} = ${value},`; | ||
} | ||
}); | ||
|
||
// `///` comments required to show up in compiled flatbuffer schemas | ||
definition = `/// ${schema.description}\nenum ${schema.name} : ubyte {\n ${fields.join( | ||
"\n\n ", | ||
)}\n}\n`; | ||
break; | ||
} | ||
case "message": { | ||
const fields = schema.fields.map((field) => { | ||
const isArray = field.array != undefined; | ||
|
||
let type; | ||
switch (field.type.type) { | ||
case "enum": | ||
type = field.type.enum.name; | ||
imports.add(field.type.enum.name); | ||
break; | ||
case "nested": | ||
type = `foxglove.${field.type.schema.name}`; | ||
imports.add(field.type.schema.name); | ||
break; | ||
case "primitive": | ||
if (field.type.name === "time") { | ||
type = "Time"; | ||
imports.add(`Time`); | ||
} else if (field.type.name === "duration") { | ||
type = "Duration"; | ||
imports.add(`Duration`); | ||
} else if (field.type.name === "bytes" && isArray) { | ||
type = "ByteVector"; | ||
imports.add("ByteVector"); | ||
} else { | ||
type = primitiveToFlatbuffers(field.type.name); | ||
} | ||
break; | ||
} | ||
let lengthComment; | ||
|
||
if (typeof field.array === "number") { | ||
// can't specify length of vector outside of struct, all of these are tables | ||
lengthComment = ` /// length ${field.array}\n`; | ||
} | ||
let defaultValue; | ||
if (field.defaultValue != undefined) { | ||
defaultValue = field.defaultValue; | ||
} | ||
|
||
return `${field.description | ||
.trim() | ||
.split("\n") | ||
.map((line) => ` /// ${line}\n`) | ||
.join("")}${ | ||
// can't have inline comments, so the lengthComment needs to be above | ||
lengthComment ?? "" | ||
// convert field.name to lowercase for flatbuffer compilation compliance | ||
} ${field.name.toLowerCase()}:${isArray ? `[${type}]` : type}${ | ||
defaultValue ? ` = ${defaultValue}` : "" | ||
};`; | ||
}); | ||
|
||
definition = `/// ${schema.description}\ntable ${schema.name} {\n${fields.join( | ||
"\n\n", | ||
)}\n}\n\nroot_type ${schema.name};`; | ||
break; | ||
} | ||
} | ||
|
||
const outputSections = [ | ||
`// Generated by https://github.com/foxglove/schemas`, | ||
|
||
Array.from(imports) | ||
.sort() | ||
.map((name) => `include "${name}.fbs";`) | ||
.join("\n"), | ||
|
||
`namespace foxglove;`, | ||
|
||
definition, | ||
].filter(Boolean); | ||
|
||
return outputSections.join("\n\n") + "\n"; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,6 +30,7 @@ export type FoxgloveMessageField = { | |
required?: true; | ||
description: string; | ||
protobufFieldNumber?: number; | ||
defaultValue?: string; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the default value for numeric fields probably shouldn't be a string, it should be the actual type of the field ( There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
}; | ||
|
||
export type FoxgloveMessageSchema = { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If there were warnings/errors would it actually affect the exit code of this flatc invocation?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
if there are errors, it returns an exit code of 1, so I believe it should exit out. On warnings, however, it still returns 0. I can add some logic to this to fail out when there's a warning though
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah I think if that's not too hard, it would be nice to catch warnings as well. (Assuming the warnings are actually useful and not just noisy/pedantic)