incrementing our metadata classes

[d-v-b]

proposal

I'm interested in improving the way zarr-python models zarr metadata documents. I think we should aim for metadata documents to be simple python classes that do little more than:

• very lightly normalize the underlying JSON, e.g. mapping "bytes" and {"name": " "bytes"} and {"name": "bytes", "configuration": {}} to the same object.
• provide an object-with-attributes API for zarr metadata

Then, the job of relating metadata to runtime functionality (like codecs) is the job of a metadata consumer (e.g., the array).

For example, instead of today's codecs: tuple[CodecClassesThatEncodeStuff, ...], we would have codecs: tuple[GenericZarrMetadataClass, ...].

This buys us a few things:

• It's easier to expose metadata classes as stable public API, because they are not coupled to runtime internals like codec classes. Instead, they are coupled to the zarr spec itself, which is more stable.
• We can re-use our metadata classes across different zarr backends (e.g., tensorstore, zarrista) when we eventually add multi-backend support.

This would be a breaking change to the .metadata attribute of the array / async array classes. We would need to create a smooth path for consumers who currently use our existing metadata classes.

API outline

I'm thinking something like this (code copy + pasted from another zarr project i'm working on):

@dataclass(frozen=True, slots=True, kw_only=True)
class ZarrMetadataV3:
    name: str
    configuration: dict[str, object]

    def to_json(self) -> MetadataFieldV3:
        return {"name": self.name, "configuration": self.configuration}

    @classmethod
    def from_json(cls, data: object) -> ZarrMetadataV3:
        field = parse_metadata_field_v3(data)
        if isinstance(field, str):
            return cls(name=field, configuration={})
        configuration = _arrays_to_tuples(dict(field.get("configuration", {})))
        return cls(name=field["name"], configuration=configuration)  # type: ignore[arg-type]

@dataclass(frozen=True, slots=True, kw_only=True)
class ArrayMetadataModelV3:
    zarr_format: Literal[3] = field(default=3, init=False)
    node_type: Literal["array"] = field(default="array", init=False)
    shape: tuple[int, ...]
    fill_value: JSON
    data_type: ZarrMetadataV3
    chunk_grid: ZarrMetadataV3
    codecs: tuple[ZarrMetadataV3, ...]
    chunk_key_encoding: ZarrMetadataV3
    dimension_names: tuple[str | None, ...] | None
    attributes: dict[str, object]
    storage_transformers: tuple[ZarrMetadataV3, ...]
    extra_fields: dict[str, ExtensionFieldV3]

    @classmethod
    def create_default(
        cls, **overrides: Unpack[ArrayMetadataModelV3Partial]
    ) -> ArrayMetadataModelV3:
        """
        Create a default (empty) v3 array metadata model, with optional overrides.

        The default is a structurally-valid scalar `uint8` array — the array
        analog of `list()` returning `[]`. Any field can be overridden by keyword
        (the same fields accepted by `update`).
        """
        default = cls(
            shape=(),
            fill_value=0,
            data_type=ZarrMetadataV3(name="uint8", configuration={}),
            chunk_grid=ZarrMetadataV3(
                name="regular", configuration={"chunk_shape": ()}
            ),
            codecs=(ZarrMetadataV3(name="bytes", configuration={}),),
            chunk_key_encoding=ZarrMetadataV3(name="default", configuration={}),
            dimension_names=None,
            attributes={},
            storage_transformers=(),
            extra_fields={},
        )
        return default.update(**overrides)

    def update(
        self, **kwargs: Unpack[ArrayMetadataModelV3Partial]
    ) -> ArrayMetadataModelV3:
        """
        Return a new `ArrayMetadataModelV3` with the given fields updated.

        Only the constructor-settable fields listed in
        `ArrayMetadataModelV3Partial` can be updated; any attempt to update
        other fields (including the fixed `zarr_format` / `node_type`) is
        rejected at the type level. Each given field fully replaces its
        previous value, including `extra_fields`.

        This is useful for test fixtures that want to override a few fields of a
        base template without having to re-specify the entire document.
        """
        return dataclasses.replace(self, **kwargs)

    def __post_init__(self):
        if set(self.extra_fields.keys()).intersection(ARRAY_METADATA_STANDARD_KEYS_V3):
            raise ValueError(
                "Extra fields cannot overlap with standard ArrayMetadataV3 fields"
            )

    def to_json(self) -> ArrayMetadataV3:
        out: ArrayMetadataV3 = {
            "zarr_format": self.zarr_format,
            "node_type": self.node_type,
            "shape": self.shape,
            "fill_value": self.fill_value,
            "data_type": self.data_type.to_json(),
            "chunk_grid": self.chunk_grid.to_json(),
            "codecs": tuple(codec.to_json() for codec in self.codecs),
            "chunk_key_encoding": self.chunk_key_encoding.to_json(),
        }
        if self.dimension_names is not None:
            out["dimension_names"] = self.dimension_names
        if len(self.attributes) > 0:
            out["attributes"] = self.attributes
        if len(self.storage_transformers) > 0:
            out["storage_transformers"] = tuple(
                transformer.to_json() for transformer in self.storage_transformers
            )
        # Extra fields are the TypedDict's `extra_items` (PEP 728). Assign them
        # by key rather than `out.update(**...)`: type checkers understand the
        # indexed-write path against `extra_items`, but not the `update(**...)`
        # overload.
        for key, value in self.extra_fields.items():
            out[key] = value
        return out

    @classmethod
    def from_json(cls, data: object) -> ArrayMetadataModelV3:
        parsed = parse_array_metadata_v3(_arrays_to_tuples(data))
        extra_fields: dict[str, ExtensionFieldV3] = {
            k: v  # type: ignore[misc]
            for k, v in parsed.items()
            if k not in ARRAY_METADATA_STANDARD_KEYS_V3
        }
        return cls(
            shape=parsed["shape"],
            fill_value=parsed["fill_value"],  # type: ignore[arg-type]  # fill_value: object in upstream TypedDict
            data_type=ZarrMetadataV3.from_json(parsed["data_type"]),
            chunk_grid=ZarrMetadataV3.from_json(parsed["chunk_grid"]),
            codecs=tuple(ZarrMetadataV3.from_json(c) for c in parsed["codecs"]),
            chunk_key_encoding=ZarrMetadataV3.from_json(parsed["chunk_key_encoding"]),
            dimension_names=parsed.get("dimension_names"),
            attributes=dict(parsed.get("attributes", {})),
            storage_transformers=tuple(
                ZarrMetadataV3.from_json(t)
                for t in parsed.get("storage_transformers", ())
            ),
            extra_fields=extra_fields,
        )

    @classmethod
    def from_key_value(cls, mapping: Mapping[str, bytes]) -> ArrayMetadataModelV3:
        return cls.from_json(json.loads(mapping["zarr.json"]))

    def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]:
        return {"zarr.json": json.dumps(self.to_json(), indent=indent).encode("utf-8")}

[d-v-b]

see #4119 for a PR that implements this on the zarr-metadata side of things. the idea is that zarr-python would phase out its own ArrayV3Metadata class, and use the classes defined in that PR instead