DRAFT: serialisation docs

DEVELOPMENT.md

@@ -110,6 +110,25 @@ just coverage
 and open it with your favourite coverage viewer. In VSCode, you can use
 [`coverage-gutters`](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters).
 
+
+## Serialisation
+
+- before you modify the schema, write a test case in
+  `hugr::serialize::upgrade::test` that exercises the part of the schema that
+  will change.
+- run the tests, this will create a file in the `testcases` subdirectory.
+
+- modify the rust types in a schema-breaking way.
+- update the `hugry-py` schema to match
+- update `get_serialisation_version` in `hugr-py/src/hugr/__init__.py`
+- rename last `V` constructor to match. (e.g. `V12` -> `V13`)
+- create new `V` constructor holding a `serde_json::Value`. (e.g. `V12(serde_json::Value)`)
+- add a stub implement `v12_to_13` in `hugr::serialize::upgrade`
+- add a line to the pattern match in `Versioned::Upgrade` e.g.  `Self::V12(json) => self = Self::V13(upgrade::v12_to_v13(json).and_then(go)?)`,
+- Ensure all non-upgrade tests are passing.
+- implement `v12_to_13` in `hugr::serialize::upgrade`
+- Ensure all tests are passing
+
 ## 🌐 Contributing to HUGR
 
 We welcome contributions to HUGR! Please open [an issue](https://github.com/CQCL/hugr/issues/new) or [pull request](https://github.com/CQCL/hugr/compare) if you have any questions or suggestions.

hugr-core/src/hugr/serialize.rs

@@ -1,6 +1,8 @@
 //! Serialization definition for [`Hugr`]
 //! [`Hugr`]: crate::hugr::Hugr
 
+use itertools::zip_eq;
+use serde::de::DeserializeOwned;
 use std::collections::HashMap;
 use thiserror::Error;
 
@@ -14,8 +16,12 @@ use portgraph::{Direction, LinkError, PortView};
 
 use serde::{Deserialize, Deserializer, Serialize};
 
+use self::upgrade::UpgradeError;
+
 use super::{HugrMut, HugrView, NodeMetadataMap};
 
+mod upgrade;
+
 /// A wrapper over the available HUGR serialization formats.
 ///
 /// The implementation of `Serialize` for `Hugr` encodes the graph in the most
@@ -27,39 +33,64 @@ use super::{HugrMut, HugrView, NodeMetadataMap};
 ///
 /// Make sure to order the variants from newest to oldest, as the deserializer
 /// will try to deserialize them in order.
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(tag = "version", rename_all = "lowercase")]
-enum Versioned<SerHugr> {
+enum Versioned<SerHugr = SerHugrLatest> {
+    #[serde(skip_serializing)]
     /// Version 0 of the HUGR serialization format.
     V0,
-    /// Version 1 of the HUGR serialization format.
-    V1(SerHugr),
 
+    V1(serde_json::Value),
+    V2(SerHugr),
+
+    #[serde(skip_serializing)]
     #[serde(other)]
     Unsupported,
 }
 
 impl<T> Versioned<T> {
-    pub fn new(t: T) -> Self {
-        Self::V1(t)
+    pub fn new_latest(t: T) -> Self {
+        Self::V2(t)
+    }
+}
+
+impl<T: DeserializeOwned> Versioned<T> {
+    fn upgrade(mut self) -> Result<T, UpgradeError> {
+        // go is polymorphic in D. When we are upgrading to the latest version
+        // D is T. When we are upgrading to a version which is not the latest D
+        // is serde_json::Value.
+        fn go<D: serde::de::DeserializeOwned>(v: serde_json::Value) -> Result<D, UpgradeError> {
+            serde_json::from_value(v).map_err(Into::into)
+        }
+        loop {
+            match self {
+                Self::V0 => Err(UpgradeError::KnownVersionUnsupported("0".into()))?,
+                // the upgrade lines remain unchanged when adding a new constructor
+                Self::V1(json) => self = Self::V2(upgrade::v1_to_v2(json).and_then(go)?),
+                Self::V2(ser_hugr) => return Ok(ser_hugr),
+                Versioned::Unsupported => Err(UpgradeError::UnknownVersionUnsupported)?,
+            }
+        }
     }
 }
 
 #[derive(Clone, Serialize, Deserialize, PartialEq, Debug)]
 struct NodeSer {
-    parent: Node,
     input_extensions: Option<ExtensionSet>,
     #[serde(flatten)]
     op: OpType,
 }
 
 /// Version 1 of the HUGR serialization format.
-#[derive(Serialize, Deserialize, PartialEq, Debug)]
-struct SerHugrV1 {
+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
+struct SerHugrLatest {
     /// For each node: (parent, node_operation)
     nodes: Vec<NodeSer>,
     /// for each edge: (src, src_offset, tgt, tgt_offset)
     edges: Vec<[(Node, Option<u16>); 2]>,
+    /// for each node: it's parent node. The root node has itself as a parent.
+    hierarchy: Vec<Node>,
+
     /// for each node: (metadata)
     #[serde(default)]
     metadata: Option<Vec<Option<NodeMetadataMap>>>,
@@ -104,8 +135,8 @@ impl Serialize for Hugr {
     where
         S: serde::Serializer,
     {
-        let shg: SerHugrV1 = self.try_into().map_err(serde::ser::Error::custom)?;
-        let versioned = Versioned::new(shg);
+        let shg: SerHugrLatest = self.try_into().map_err(serde::ser::Error::custom)?;
+        let versioned = Versioned::new_latest(shg);
         versioned.serialize(serializer)
     }
 }
@@ -115,20 +146,13 @@ impl<'de> Deserialize<'de> for Hugr {
     where
         D: Deserializer<'de>,
     {
-        let shg: Versioned<SerHugrV1> = Versioned::deserialize(deserializer)?;
-        match shg {
-            Versioned::V0 => Err(serde::de::Error::custom(
-                "Version 0 HUGR serialization format is not supported.",
-            )),
-            Versioned::V1(shg) => shg.try_into().map_err(serde::de::Error::custom),
-            Versioned::Unsupported => Err(serde::de::Error::custom(
-                "Unsupported HUGR serialization format.",
-            )),
-        }
+        let versioned = Versioned::deserialize(deserializer)?;
+        let shl: SerHugrLatest = versioned.upgrade().map_err(serde::de::Error::custom)?;
+        shl.try_into().map_err(serde::de::Error::custom)
     }
 }
 
-impl TryFrom<&Hugr> for SerHugrV1 {
+impl TryFrom<&Hugr> for SerHugrLatest {
     type Error = HUGRSerializationError;
 
     fn try_from(hugr: &Hugr) -> Result<Self, Self::Error> {
@@ -141,22 +165,29 @@ impl TryFrom<&Hugr> for SerHugrV1 {
 
         let mut nodes = vec![None; hugr.node_count()];
         let mut metadata = vec![None; hugr.node_count()];
+        let mut hierarchy = vec![None; hugr.node_count()];
         for n in hugr.nodes() {
             let parent = node_rekey[&hugr.get_parent(n).unwrap_or(n)];
             let opt = hugr.get_nodetype(n);
             let new_node = node_rekey[&n].index();
             nodes[new_node] = Some(NodeSer {
-                parent,
                 input_extensions: opt.input_extensions.clone(),
                 op: opt.op.clone(),
             });
+
+            hierarchy[new_node] = Some(parent);
             metadata[new_node].clone_from(hugr.metadata.get(n.pg_index()));
         }
         let nodes = nodes
             .into_iter()
             .collect::<Option<Vec<_>>>()
             .expect("Could not reach one of the nodes");
 
+        let hierarchy = hierarchy
+            .into_iter()
+            .collect::<Option<Vec<_>>>()
+            .expect("One of the nodes is missing a parent");
+
         let find_offset = |node: Node, offset: usize, dir: Direction, hugr: &Hugr| {
             let op = hugr.get_optype(node);
             let is_value_port = offset < op.value_port_count(dir);
@@ -185,43 +216,48 @@ impl TryFrom<&Hugr> for SerHugrV1 {
         Ok(Self {
             nodes,
             edges,
+            hierarchy,
             metadata: Some(metadata),
             encoder,
         })
     }
 }
 
-impl TryFrom<SerHugrV1> for Hugr {
+impl TryFrom<SerHugrLatest> for Hugr {
     type Error = HUGRSerializationError;
     fn try_from(
-        SerHugrV1 {
+        SerHugrLatest {
             nodes,
             edges,
             metadata,
-            ..
-        }: SerHugrV1,
+            hierarchy,
+            encoder: _,
+        }: SerHugrLatest,
     ) -> Result<Self, Self::Error> {
+        let nodes_len = nodes.len();
         // Root must be first node
-        let mut nodes = nodes.into_iter();
-        let NodeSer {
-            parent: root_parent,
-            input_extensions,
-            op: root_type,
-        } = nodes.next().unwrap();
-        if root_parent.index() != 0 {
-            return Err(HUGRSerializationError::FirstNodeNotRoot(root_parent));
+        let mut nodes_parent = zip_eq(nodes, hierarchy);
+        let (
+            NodeSer {
+                input_extensions,
+                op: root_type,
+            },
+            parent,
+        ) = nodes_parent.next().unwrap();
+        if parent.index() != 0 {
+            return Err(HUGRSerializationError::FirstNodeNotRoot(parent));
         }
         // if there are any unconnected ports or copy nodes the capacity will be
         // an underestimate
         let mut hugr = Hugr::with_capacity(
             NodeType::new(root_type, input_extensions),
-            nodes.len(),
+            nodes_len,
             edges.len() * 2,
         );
 
-        for node_ser in nodes {
+        for (node_ser, parent) in nodes_parent {
             hugr.add_node_with_parent(
-                node_ser.parent,
+                parent,
                 NodeType::new(node_ser.op, node_ser.input_extensions),
             );
         }