Add Flatbuffer Schema Support (#70)

- Add code to generate flatbuffer schema files: `.fbs`.
 - Added test for expected output

To test compilation: install `cmake` and `flatbuffer` (via homebrew or
other means) then run this command from the repo root directory: `flatc
--ts -o ./schemas/flatbuffer/output
./schemas/flatbuffer/foxglove/**.fbs`
and see that it generates the schema files in the
`schemas/flatbuffer/output` directory.

Co-authored-by: Jacob Bandes-Storch <jacob@foxglove.dev>

.github/workflows/ci.yml

@@ -36,6 +36,18 @@ jobs:
             echo "Generated schemas are up to date!"
           fi
 
+      - name: Validate Flatbuffer definitions
+        run: |
+          curl -LO https://github.com/google/flatbuffers/releases/download/v22.10.26/Linux.flatc.binary.clang++-12.zip 
+          echo "0821af82a3a736b0ba9235c02219df24d1f042dd  Linux.flatc.binary.clang++-12.zip" | shasum -a 1 -c
+          unzip Linux.flatc.binary.clang++-12.zip
+          output=$(./flatc --ts -o /dev/null ./schemas/flatbuffer/*.fbs)
+          if [ -n "$output" ]; then
+            echo "::error::Flatbuffer schema compilation had warnings or errors. Fix them to proceed:"
+            echo "$output"
+            exit 1
+          fi
+
       - name: Validate protobuf definitions
         run: protoc --proto_path=schemas/proto schemas/proto/**/*.proto --descriptor_set_out=/dev/null
 

internal/generateFlatbufferSchema.test.ts

@@ -0,0 +1,118 @@
+import { generateFlatbuffers } from "./generateFlatbufferSchema";
+import { exampleEnum, exampleMessage } from "./testFixtures";
+
+describe("generateFlatbuffers", () => {
+  it("generates Message .fbs files", () => {
+    expect(generateFlatbuffers(exampleMessage, [exampleEnum])).toMatchInlineSnapshot(`
+      "// Generated by https://github.com/foxglove/schemas
+
+      include "ByteVector.fbs";
+      include "Duration.fbs";
+      include "NestedMessage.fbs";
+      include "Time.fbs";
+
+      namespace foxglove;
+
+      /// An example enum
+      enum ExampleEnum : ubyte {
+        /// Value A
+        A = 1,
+
+        /// Value B
+        B = 2,
+      }
+      /// An example type
+      table ExampleMessage {
+        /// duration field
+        field_duration:Duration;
+
+        /// time field
+        field_time:Time;
+
+        /// boolean field
+        field_boolean:bool = true;
+
+        /// bytes field
+        field_bytes:[uint8];
+
+        /// float64 field
+        field_float64:double = 1.0;
+
+        /// uint32 field
+        field_uint32:uint32 = 5;
+
+        /// string field
+        field_string:string = "string-type";
+
+        /// duration array field
+        field_duration_array:[Duration];
+
+        /// time array field
+        field_time_array:[Time];
+
+        /// boolean array field
+        field_boolean_array:[bool];
+
+        /// bytes array field
+        field_bytes_array:[ByteVector];
+
+        /// float64 array field
+        field_float64_array:[double];
+
+        /// uint32 array field
+        field_uint32_array:[uint32];
+
+        /// string array field
+        field_string_array:[string];
+
+        /// duration fixed-length array field
+        /// length 3
+        field_duration_fixed_array:[Duration];
+
+        /// time fixed-length array field
+        /// length 3
+        field_time_fixed_array:[Time];
+
+        /// boolean fixed-length array field
+        /// length 3
+        field_boolean_fixed_array:[bool];
+
+        /// bytes fixed-length array field
+        /// length 3
+        field_bytes_fixed_array:[ByteVector];
+
+        /// float64 fixed-length array field
+        /// length 3
+        field_float64_fixed_array:[double];
+
+        /// uint32 fixed-length array field
+        /// length 3
+        field_uint32_fixed_array:[uint32];
+
+        /// string fixed-length array field
+        /// length 3
+        field_string_fixed_array:[string];
+
+        /// An enum field
+        field_enum:ExampleEnum;
+
+        /// An enum array field
+        field_enum_array:[ExampleEnum];
+
+        /// A nested field
+        field_nested:foxglove.NestedMessage;
+
+        /// A nested array field
+        /// With
+        /// a
+        /// very
+        /// long
+        /// description
+        field_nested_array:[foxglove.NestedMessage];
+      }
+
+      root_type ExampleMessage;
+      "
+    `);
+  });
+});

internal/generateFlatbufferSchema.ts

@@ -0,0 +1,189 @@
+import { FoxgloveEnumSchema, FoxglovePrimitive, FoxgloveSchema } from "./types";
+
+// Flatbuffers only supports nested vectors via table
+export const BYTE_VECTOR_FB = `
+namespace foxglove;
+
+/// Used for nesting byte vectors
+table ByteVector {
+  data:[uint8];
+}
+root_type ByteVector;
+`;
+
+// Same as protobuf wellknown types
+export const TIME_FB = `
+namespace foxglove;
+
+struct Time {
+  /// Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z
+  sec:uint32;
+  /// Nano-second fractions from 0 to 999,999,999 inclusive
+  nsec:uint32;
+}
+`;
+
+export const DURATION_FB = `
+namespace foxglove;
+
+struct Duration {
+  /// Signed seconds of the span of time. Must be from -315,576,000,000 to +315,576,000,000 inclusive.
+  sec:int32;
+  /// if sec === 0 : -999,999,999 <= nsec <= +999,999,999 
+  /// otherwise sign of sec must match sign of nsec or be 0 and abs(nsec) <= 999,999,999
+  nsec:int32;
+}
+`;
+
+function primitiveToFlatbuffers(type: Exclude<FoxglovePrimitive, "time" | "duration">) {
+  switch (type) {
+    case "uint32":
+      return "uint32";
+    case "bytes":
+      return "[uint8]";
+    case "string":
+      return "string";
+    case "boolean":
+      return "bool";
+    case "float64":
+      return "double";
+  }
+}
+
+export function generateFlatbuffers(
+  schema: FoxgloveSchema,
+  nestedEnums: FoxgloveEnumSchema[],
+): string {
+  const enumDefinitions: string[] = [];
+  for (const enumSchema of nestedEnums) {
+    const fields = enumSchema.values.map(({ name, value, description }) => {
+      if (description != undefined) {
+        return `/// ${description}\n  ${name} = ${value},`;
+      } else {
+        return `${name} = ${value},`;
+      }
+    });
+    enumDefinitions.push(
+      // `///` comments required to show up in compiled flatbuffer schemas
+      `/// ${enumSchema.description}\nenum ${enumSchema.name} : ubyte {\n  ${fields.join(
+        "\n\n  ",
+      )}\n}\n`,
+    );
+  }
+
+  let definition;
+  const imports = new Set<string>();
+  switch (schema.type) {
+    case "enum": {
+      const fields = schema.values.map(({ name, value, description }) => {
+        if (description != undefined) {
+          return `/// ${description}\n  ${name} = ${value},`;
+        } else {
+          return `${name} = ${value},`;
+        }
+      });
+
+      // `///` comments required to show up in compiled flatbuffer schemas
+      definition = `/// ${schema.description}\nenum ${schema.name} : ubyte {\n  ${fields.join(
+        "\n\n  ",
+      )}\n}\n`;
+      break;
+    }
+    case "message": {
+      const fields = schema.fields.map((field) => {
+        const isArray = field.array != undefined;
+
+        let type;
+        switch (field.type.type) {
+          case "enum":
+            type = field.type.enum.name;
+            break;
+          case "nested":
+            type = `foxglove.${field.type.schema.name}`;
+            imports.add(field.type.schema.name);
+            break;
+          case "primitive":
+            if (field.type.name === "time") {
+              type = "Time";
+              imports.add(`Time`);
+            } else if (field.type.name === "duration") {
+              type = "Duration";
+              imports.add(`Duration`);
+            } else if (field.type.name === "bytes" && isArray) {
+              type = "ByteVector";
+              imports.add("ByteVector");
+            } else {
+              type = primitiveToFlatbuffers(field.type.name);
+            }
+            break;
+        }
+        let lengthComment;
+
+        if (typeof field.array === "number") {
+          // can't specify length of vector outside of struct, all of these are tables
+          lengthComment = `  /// length ${field.array}\n`;
+        }
+        let defaultValue;
+        if (field.defaultValue != undefined && !isArray) {
+          if (
+            field.type.type === "primitive" &&
+            !(field.type.name === "duration" || field.type.name === "time")
+          ) {
+            if (typeof field.defaultValue === "string") {
+              defaultValue = `"${field.defaultValue}"`;
+            } else if (typeof field.defaultValue === "number") {
+              if (Number.isInteger(field.defaultValue) && field.type.name === "float64") {
+                // if it is a floating point number that is an integer, we need to add a decimal point
+                defaultValue = `${field.defaultValue}.0`;
+              } else {
+                defaultValue = field.defaultValue.toString();
+              }
+            } else if (typeof field.defaultValue === "boolean") {
+              // uses same 'false'/'true' as js
+              defaultValue = field.defaultValue.toString();
+            }
+          } else if (field.type.type === "enum") {
+            // default enums are just the enum string of the enum and don't require other formatting
+            // ie: type numericType: NumericType = INT32;
+            defaultValue = field.defaultValue as string;
+          }
+        }
+        if (field.defaultValue != undefined && defaultValue == undefined) {
+          throw new Error("Flatbuffers does not support non-scalar default values");
+        }
+
+        return `${field.description
+          .trim()
+          .split("\n")
+          .map((line) => `  /// ${line}\n`)
+          .join("")}${
+          // can't have inline comments, so the lengthComment needs to be above
+          lengthComment ?? ""
+          // convert field.name to lowercase for flatbuffer compilation compliance
+        }  ${field.name.toLowerCase()}:${isArray ? `[${type}]` : type}${
+          defaultValue ? ` = ${defaultValue}` : ""
+        };`;
+      });
+
+      definition = `${enumDefinitions.join("\n\n")}/// ${schema.description}\ntable ${
+        schema.name
+      } {\n${fields.join("\n\n")}\n}\n\nroot_type ${schema.name};`;
+      break;
+    }
+  }
+
+  const outputSections = [
+    `// Generated by https://github.com/foxglove/schemas`,
+
+    Array.from(imports)
+      .sort()
+      .map((name) => `include "${name}.fbs";`)
+      .join("\n"),
+
+    `namespace foxglove;`,
+
+    definition,
+  ].filter(Boolean);
+
+  return outputSections.join("\n\n") + "\n";
+}

internal/generateProto.test.ts

@@ -117,7 +117,7 @@ describe("generateProto", () => {
     root.addJSON(protobufjs.common.get("google/protobuf/duration.proto")!.nested!);
     for (const schema of Object.values(foxgloveMessageSchemas)) {
       const enums = Object.values(foxgloveEnumSchemas).filter(
-        (enumSchema) => enumSchema.protobufParentMessageName === schema.name,
+        (enumSchema) => enumSchema.parentSchemaName === schema.name,
       );
       root.add(protobufjs.parse(generateProto(schema, enums)).root);
     }

internal/schemas.test.ts

@@ -8,7 +8,7 @@ describe("schemas", () => {
     }
     for (const [key, value] of Object.entries(foxgloveEnumSchemas)) {
       expect(key).toEqual(value.name);
-      expect(value.protobufParentMessageName in foxgloveMessageSchemas).toBe(true);
+      expect(value.parentSchemaName in foxgloveMessageSchemas).toBe(true);
     }
   });