Shared Tree: Schema FormatV2 and codecs (#24812)

Adds support for persisting metadata along with the schema.

# Changes

- Add schema FormatV2
   - Node schema persisted metadata
  - Field schema persisted metadata
- Schema codec for FormatV2
  - Add encodeV2 method to TreeNodeStoredSchema hierarchy
  - Persist node schema metadata in TreeNodeStoredSchema
  - Add support for encoding v2 fields
- Update the storedSchemaDecodeDispatcher to dispatch on the inner kind
union in FormatV2 and pass node schema metadata
  - Create and register schema change codec handling FormatV2
  - Create and register schema codec handling FormatV2
  - Generate new snapshots for schema tests
- Add a new Shared Tree format version
  - Update enums
  - Generate new snapshots for existing Shared Tree format tests
- simple-tree changes
  - Persist node metadata in arraySchema, mapSchema, objectSchema
  - Pass metadata to NodeStoredSchema types in toStoredSchema.ts
- Other tests
  - Update codec.spec to cover FormatV2 as well as FormatV1 
  - FormatV2-specific tests
- API documentation
  - Changeset covering the new APIs
- Feature flag for enabling Shared Tree Format v5

---------

Co-authored-by: Craig Macomber (Microsoft) <42876482+CraigMacomber@users.noreply.github.com>
Co-authored-by: Alex Villarreal <716334+alexvy86@users.noreply.github.com>
Co-authored-by: Abram Sanderson <Abram.sanderson@gmail.com>

Author: 4 people

.changeset/spotty-bottles-rest.md

@@ -0,0 +1,109 @@
+---
+"fluid-framework": minor
+"@fluidframework/tree": minor
+"__section": feature
+---
+Persisted metadata for Shared Tree schemas (Alpha)
+
+The persisted metadata feature for Shared Tree allows an application author to write document-persisted metadata along with the schema. This feature is supported for both node and field schemas.
+
+#### Using the persisted metadata feature
+
+As of now, persisted metadata support is available via the SchemaFactoryAlpha API:
+
+```ts
+// Construct a schema factory with alpha APIs
+const schemaFactory = new SchemaFactoryAlpha("com.example");
+```
+
+Persisted metadata can take the shape of any JSON-serializable object, e.g.:
+
+```ts
+const persistedMetadata = { a: 2 };
+```
+
+#### Feature flag
+
+To enable persisted metadata, use `configuredSharedTree` to specify the format version. The tree that is returned can be substituted in place of the default `SharedTree` object exported by the Fluid Framework. For example:
+
+```ts
+const tree = configuredSharedTree({
+    formatVersion: SharedTreeFormatVersion.v5,
+}).create(runtime);
+
+export const MyContainerSchema = {
+    initialObjects: {
+        appData: tree,
+    },
+} satisfies ContainerSchema;
+```
+
+#### Examples
+
+##### Field schemas with persisted metadata
+
+```ts
+// Construct a schema factory with alpha APIs
+const schemaFactory = new SchemaFactoryAlpha("com.example");
+
+// Define metadata. This can take the shape of any JSON-serializable object.
+const persistedMetadata = { "a": 2 };
+
+// Foo is an object type with metadata
+class Foo extends schemaFactory.objectAlpha("Foo", {
+    // Metadata for a required number field
+    bar: schemaFactory.required(schemaFactory.number, { persistedMetadata }),
+
+    // Metadata for an optional string field   
+    baz: schemaFactory.optional(schemaFactory.string, { persistedMetadata }),
+// Metadata for the object type Foo       
+}, { persistedMetadata }) {}
+```
+
+##### Recursive field schemas
+
+```ts
+// Construct a schema factory with alpha APIs
+const schemaFactory = new SchemaFactoryAlpha("com.example");
+
+// Define metadata. This can take the shape of any JSON-serializable object.
+const persistedMetadata = { "a": 2 };
+
+// Recursive object schema with persisted metadata
+class RecursiveObject extends schemaFactory.objectRecursive("RecursiveObject", {
+    x: [() => RecursiveObject, schemaFactory.number],
+}, { persistedMetadata }) {}
+
+// Recursive field schema with metadata
+const recursiveField = schemaFactory.optionalRecursive(
+    [() => RecursiveObject, schemaFactory.number],
+    { persistedMetadata });
+```
+
+##### Recursive object schemas
+
+```ts
+// Construct a schema factory with alpha APIs
+const schemaFactory = new SchemaFactoryAlpha("com.example");
+
+// Define metadata. This can take the shape of any JSON-serializable object.
+const persistedMetadata = { "a": 2 };
+
+// Recursive array schema
+class Foos extends schemaFactory.arrayRecursive(
+    "FooList",
+    [() => Foo],
+    { persistedMetadata }) {}
+
+// Recursive object schema
+class Foo extends schemaFactory.objectRecursive(
+    "Foo",
+    { fooList: Foos },
+    { persistedMetadata }) {}
+
+// Recursive map schema
+class FooMap extends schemaFactory.mapRecursive(
+    "FooMap",
+    [() => Foo],
+    { persistedMetadata }) {}
+```

.changeset/whole-days-argue.md

@@ -858,6 +858,7 @@ export const SharedTreeFormatVersion: {
     readonly v1: 1;
     readonly v2: 2;
     readonly v3: 3;
+    readonly v5: 5;
 };
 
 // @alpha

packages/dds/tree/api-report/tree.alpha.api.md

@@ -138,11 +138,13 @@ export {
 	storedEmptyFieldSchema,
 	type StoredSchemaCollection,
 	schemaFormatV1,
+	schemaFormatV2,
 	LeafNodeStoredSchema,
 	ObjectNodeStoredSchema,
 	MapNodeStoredSchema,
 	decodeFieldSchema,
-	encodeFieldSchema,
+	encodeFieldSchemaV1,
+	encodeFieldSchemaV2,
 	storedSchemaDecodeDispatcher,
 	type SchemaAndPolicy,
 	Multiplicity,

packages/dds/tree/src/core/index.ts

@@ -0,0 +1,78 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { type ObjectOptions, type Static, Type } from "@sinclair/typebox";
+
+import { JsonCompatibleReadOnlySchema } from "../../util/index.js";
+import {
+	FieldKindIdentifierSchema,
+	PersistedValueSchema,
+	TreeNodeSchemaIdentifierSchema,
+} from "./formatV1.js";
+import { unionOptions } from "../../codec/index.js";
+
+export const PersistedMetadataFormat = Type.Optional(JsonCompatibleReadOnlySchema);
+
+const FieldSchemaFormatBase = Type.Object({
+	kind: FieldKindIdentifierSchema,
+	types: Type.Array(TreeNodeSchemaIdentifierSchema),
+	metadata: PersistedMetadataFormat,
+});
+
+const noAdditionalProps: ObjectOptions = { additionalProperties: false };
+
+export const FieldSchemaFormat = Type.Composite([FieldSchemaFormatBase], noAdditionalProps);
+
+/**
+ * Format for the content of a {@link TreeNodeStoredSchema}.
+ *
+ * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
+ */
+export const TreeNodeSchemaUnionFormat = Type.Object(
+	{
+		/**
+		 * Object node union member.
+		 */
+		object: Type.Optional(Type.Record(Type.String(), FieldSchemaFormat)),
+		/**
+		 * Map node union member.
+		 */
+		map: Type.Optional(FieldSchemaFormat),
+		/**
+		 * Leaf node union member.
+		 */
+		leaf: Type.Optional(Type.Enum(PersistedValueSchema)),
+	},
+	unionOptions,
+);
+
+export type TreeNodeSchemaUnionFormat = Static<typeof TreeNodeSchemaUnionFormat>;
+
+/**
+ * Format for {@link TreeNodeStoredSchema}.
+ *
+ * See {@link DiscriminatedUnionDispatcher} for more information on this pattern.
+ */
+export const TreeNodeSchemaDataFormat = Type.Object(
+	{
+		/**
+		 * Node kind specific data.
+		 */
+		kind: TreeNodeSchemaUnionFormat,
+
+		// Data in common for all TreeNode schemas:
+		/**
+		 * Leaf node union member.
+		 */
+		metadata: PersistedMetadataFormat,
+	},
+	noAdditionalProps,
+);
+
+export type TreeNodeSchemaDataFormat = Static<typeof TreeNodeSchemaDataFormat>;
+
+export type FieldSchemaFormat = Static<typeof FieldSchemaFormat>;
+
+export type PersistedMetadataFormat = Static<typeof PersistedMetadataFormat>;

packages/dds/tree/src/core/schema-stored/formatV2.ts

@@ -18,7 +18,8 @@ export {
 	ObjectNodeStoredSchema,
 	MapNodeStoredSchema,
 	decodeFieldSchema,
-	encodeFieldSchema,
+	encodeFieldSchemaV1,
+	encodeFieldSchemaV2,
 	storedSchemaDecodeDispatcher,
 	type SchemaAndPolicy,
 	type SchemaPolicy,
@@ -37,3 +38,5 @@ export type { TreeNodeSchemaIdentifier, FieldKey, FieldKindIdentifier } from "./
 
 import * as schemaFormatV1 from "./formatV1.js";
 export { schemaFormatV1 };
+import * as schemaFormatV2 from "./formatV2.js";
+export { schemaFormatV2 };