Add Flatbuffer Schema Support

.github/workflows/ci.yml

@@ -36,6 +36,13 @@ jobs:
             echo "Generated schemas are up to date!"
           fi
 
+      - name: Validate Flatbuffer definitions
+        run: |
+          curl -LO https://github.com/google/flatbuffers/releases/download/v22.10.26/Linux.flatc.binary.clang++-12.zip 
+          echo "0821af82a3a736b0ba9235c02219df24d1f042dd  Linux.flatc.binary.clang++-12.zip" | shasum -a 1 -c
+          unzip Linux.flatc.binary.clang++-12.zip
+          ./flatc --ts -o /dev/null schemas/flatbuffers/*.fbs
+
       - name: Validate protobuf definitions
         run: protoc --proto_path=schemas/proto schemas/proto/**/*.proto --descriptor_set_out=/dev/null
 

internal/generateFlatbuffers.test.ts

@@ -0,0 +1,118 @@
+import { generateFlatbuffers } from "./generateFlatbuffers";
+import { exampleEnum, exampleMessage } from "./testFixtures";
+
+describe("generateFlatBuffer", () => {
+  it("generates Message .fbs files", () => {
+    expect(generateFlatbuffers(exampleMessage, [exampleEnum])).toMatchInlineSnapshot(`
+      "// Generated by https://github.com/foxglove/schemas
+
+      include "ByteVector.fbs";
+      include "Duration.fbs";
+      include "NestedMessage.fbs";
+      include "Time.fbs";
+
+      namespace foxglove;
+
+      /// An example enum
+      enum ExampleEnum : ubyte {
+        /// Value A
+        A = 1,
+
+        /// Value B
+        B = 2,
+      }
+      /// An example type
+      table ExampleMessage {
+        /// duration field
+        field_duration:Duration = 1;
+
+        /// time field
+        field_time:Time = 1;
+
+        /// boolean field
+        field_boolean:bool = true;
+
+        /// bytes field
+        field_bytes:[uint8] = 1;
+
+        /// float64 field
+        field_float64:double = 1;
+
+        /// uint32 field
+        field_uint32:uint32 = 1;
+
+        /// string field
+        field_string:string = 1;
+
+        /// duration array field
+        field_duration_array:[Duration];
+
+        /// time array field
+        field_time_array:[Time];
+
+        /// boolean array field
+        field_boolean_array:[bool];
+
+        /// bytes array field
+        field_bytes_array:[ByteVector];
+
+        /// float64 array field
+        field_float64_array:[double];
+
+        /// uint32 array field
+        field_uint32_array:[uint32];
+
+        /// string array field
+        field_string_array:[string];
+
+        /// duration fixed-length array field
+        /// length 3
+        field_duration_fixed_array:[Duration];
+
+        /// time fixed-length array field
+        /// length 3
+        field_time_fixed_array:[Time];
+
+        /// boolean fixed-length array field
+        /// length 3
+        field_boolean_fixed_array:[bool];
+
+        /// bytes fixed-length array field
+        /// length 3
+        field_bytes_fixed_array:[ByteVector];
+
+        /// float64 fixed-length array field
+        /// length 3
+        field_float64_fixed_array:[double];
+
+        /// uint32 fixed-length array field
+        /// length 3
+        field_uint32_fixed_array:[uint32];
+
+        /// string fixed-length array field
+        /// length 3
+        field_string_fixed_array:[string];
+
+        /// An enum field
+        field_enum:ExampleEnum;
+
+        /// An enum array field
+        field_enum_array:[ExampleEnum];
+
+        /// A nested field
+        field_nested:foxglove.NestedMessage;
+
+        /// A nested array field
+        /// With
+        /// a
+        /// very
+        /// long
+        /// description
+        field_nested_array:[foxglove.NestedMessage];
+      }
+
+      root_type ExampleMessage;
+      "
+    `);
+  });
+});

internal/generateFlatbuffers.ts

@@ -0,0 +1,165 @@
+import { FoxgloveEnumSchema, FoxglovePrimitive, FoxgloveSchema } from "./types";
+
+// Flatbuffers only supports nested vectors via table
+export const BYTE_VECTOR_FB = `
+namespace foxglove;
+
+/// Used for nesting byte vectors
+table ByteVector {
+  data:[uint8];
+}
+root_type ByteVector;
+`;
+
+// Same as protobuf wellknown types
+export const TIME_FB = `
+namespace foxglove;
+
+struct Time {
+  /// 	Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z
+  sec:uint64;
+  /// Nano-second fractions from 0 to 999,999,999 inclusive
+  nsec:uint32;
+}
+`;
+
+export const DURATION_FB = `
+namespace foxglove;
+
+struct Duration {
+  /// Signed seconds of the span of time. Must be from -315,576,000,000 to +315,576,000,000 inclusive.
+  sec:int64;
+  /// if sec === 0 : -999,999,999 <= nsec <= +999,999,999 
+  /// otherwise sign of sec must match sign of nsec or be 0 and abs(nsec) <= 999,999,999
+  nsec:int32;
+}
+`;
+
+function primitiveToFlatbuffers(type: Exclude<FoxglovePrimitive, "time" | "duration">) {
+  switch (type) {
+    case "uint32":
+      return "uint32";
+    case "bytes":
+      return "[uint8]";
+    case "string":
+      return "string";
+    case "boolean":
+      return "bool";
+    case "float64":
+      return "double";
+  }
+}
+
+export function generateFlatbuffers(
+  schema: FoxgloveSchema,
+  nestedEnums: FoxgloveEnumSchema[],
+): string {
+  const enumDefinitions: string[] = [];
+  for (const enumSchema of nestedEnums) {
+    const fields = enumSchema.values.map(({ name, value, description }) => {
+      if (description != undefined) {
+        return `/// ${description}\n  ${name} = ${value},`;
+      } else {
+        return `${name} = ${value},`;
+      }
+    });
+    enumDefinitions.push(
+      // `///` comments required to show up in compiled flatbuffer schemas
+      `/// ${enumSchema.description}\nenum ${enumSchema.name} : ubyte {\n  ${fields.join(
+        "\n\n  ",
+      )}\n}\n`,
+    );
+  }
+
+  let definition;
+  const imports = new Set<string>();
+  switch (schema.type) {
+    case "enum": {
+      const fields = schema.values.map(({ name, value, description }) => {
+        if (description != undefined) {
+          return `/// ${description}\n  ${name} = ${value},`;
+        } else {
+          return `${name} = ${value},`;
+        }
+      });
+
+      // `///` comments required to show up in compiled flatbuffer schemas
+      definition = `/// ${schema.description}\nenum ${schema.name} : ubyte {\n  ${fields.join(
+        "\n\n  ",
+      )}\n}\n`;
+      break;
+    }
+    case "message": {
+      const fields = schema.fields.map((field) => {
+        const isArray = field.array != undefined;
+
+        let type;
+        switch (field.type.type) {
+          case "enum":
+            type = field.type.enum.name;
+            break;
+          case "nested":
+            type = `foxglove.${field.type.schema.name}`;
+            imports.add(field.type.schema.name);
+            break;
+          case "primitive":
+            if (field.type.name === "time") {
+              type = "Time";
+              imports.add(`Time`);
+            } else if (field.type.name === "duration") {
+              type = "Duration";
+              imports.add(`Duration`);
+            } else if (field.type.name === "bytes" && isArray) {
+              type = "ByteVector";
+              imports.add("ByteVector");
+            } else {
+              type = primitiveToFlatbuffers(field.type.name);
+            }
+            break;
+        }
+        let lengthComment;
+
+        if (typeof field.array === "number") {
+          // can't specify length of vector outside of struct, all of these are tables
+          lengthComment = `  /// length ${field.array}\n`;
+        }
+        let defaultValue;
+        if (field.defaultValue != undefined) {
+          defaultValue = field.defaultValue;
+        }
+
+        return `${field.description
+          .trim()
+          .split("\n")
+          .map((line) => `  /// ${line}\n`)
+          .join("")}${
+          // can't have inline comments, so the lengthComment needs to be above
+          lengthComment ?? ""
+          // convert field.name to lowercase for flatbuffer compilation compliance
+        }  ${field.name.toLowerCase()}:${isArray ? `[${type}]` : type}${
+          defaultValue ? ` = ${defaultValue}` : ""
+        };`;
+      });
+
+      definition = `${enumDefinitions.join("\n\n")}/// ${schema.description}\ntable ${
+        schema.name
+      } {\n${fields.join("\n\n")}\n}\n\nroot_type ${schema.name};`;
+      break;
+    }
+  }
+
+  const outputSections = [
+    `// Generated by https://github.com/foxglove/schemas`,
+
+    Array.from(imports)
+      .sort()
+      .map((name) => `include "${name}.fbs";`)
+      .join("\n"),
+
+    `namespace foxglove;`,
+
+    definition,
+  ].filter(Boolean);
+
+  return outputSections.join("\n\n") + "\n";
+}

internal/schemas.ts

@@ -9,21 +9,25 @@ const Color: FoxgloveMessageSchema = {
       name: "r",
       type: { type: "primitive", name: "float64" },
       description: "Red value between 0 and 1",
+      defaultValue: "1.0",
     },
     {
       name: "g",
       type: { type: "primitive", name: "float64" },
       description: "Green value between 0 and 1",
+      defaultValue: "1.0",
     },
     {
       name: "b",
       type: { type: "primitive", name: "float64" },
       description: "Blue value between 0 and 1",
+      defaultValue: "1.0",
     },
     {
       name: "a",
       type: { type: "primitive", name: "float64" },
       description: "Alpha value between 0 and 1",
+      defaultValue: "1.0",
     },
   ],
 };
@@ -37,11 +41,13 @@ const Vector2: FoxgloveMessageSchema = {
       name: "x",
       type: { type: "primitive", name: "float64" },
       description: "x coordinate length",
+      defaultValue: "1.0",
     },
     {
       name: "y",
       type: { type: "primitive", name: "float64" },
       description: "y coordinate length",
+      defaultValue: "1.0",
     },
   ],
 };
@@ -56,16 +62,19 @@ const Vector3: FoxgloveMessageSchema = {
       name: "x",
       type: { type: "primitive", name: "float64" },
       description: "x coordinate length",
+      defaultValue: "1.0",
     },
     {
       name: "y",
       type: { type: "primitive", name: "float64" },
       description: "y coordinate length",
+      defaultValue: "1.0",
     },
     {
       name: "z",
       type: { type: "primitive", name: "float64" },
       description: "z coordinate length",
+      defaultValue: "1.0",
     },
   ],
 };
@@ -137,6 +146,7 @@ const Quaternion: FoxgloveMessageSchema = {
       name: "w",
       type: { type: "primitive", name: "float64" },
       description: "w value",
+      defaultValue: "1.0",
     },
   ],
 };

internal/testFixtures.ts

@@ -44,6 +44,7 @@ export const exampleMessage: FoxgloveMessageSchema = {
       name: `field_${name}`,
       description: `${name} field`,
       type: { type: "primitive", name },
+      defaultValue: name === "boolean" ? "true" : "1",
     })),
     ...allPrimitives.map((name): FoxgloveMessageSchema["fields"][0] => ({
       name: `field_${name}_array`,

internal/types.ts

@@ -30,6 +30,7 @@ export type FoxgloveMessageField = {
   required?: true;
   description: string;
   protobufFieldNumber?: number;
+  defaultValue?: string;
 };
 
 export type FoxgloveMessageSchema = {