Outline of structs as a general 'definition' system (#7069)

## Usage and product changes

We introduce the architecture around structs definitions, and
definitions in general.

The idea of a `definition` is that it is a general schema construct
outside the regular `thing` and `type` system - this will initially be
used for structs and functions.

Definitions are interesting because their interesting payload is in the
value, not the key. However, the key contains the ID of the definition.
As a result, you can consider the new `DefinitionKey` to be analogous to
`TypeVertex`, but the interesting data structure is the
`StructDefinition`, which serialises a struct's content.

In doing this, we also realise that `ValueType` must encompass the
built-in value type category, plus the ID of the struct definition, and
create a new `ValueTypeCategory` which can map to a Prefix and doesn't
contain the ID of struct definitions. Lastly, we create a the serialised
form of `ValueTypeBytes`, which is used as the property of an attribute
type's value type, including the definition ID if it exists.


### What isn't implemented yet:

1. The indexing + retrieval by name of value types through the
TypeManager
2. Serialisation of `StructDefinition`
3. The data insertion and deletion pattern of struct instantiations

concept/thing/attribute.rs

@@ -43,10 +43,6 @@ impl<'a> Attribute<'a> {
         Attribute { vertex, value: None }
     }
 
-    pub(crate) fn value_type(&self) -> ValueType {
-        self.vertex.value_type()
-    }
-
     pub fn type_(&self) -> AttributeType<'static> {
         AttributeType::new(build_vertex_attribute_type(self.vertex.type_id_()))
     }

concept/thing/mod.rs

@@ -6,13 +6,14 @@
 
 use bytes::byte_array::ByteArray;
 use encoding::{graph::thing::vertex_attribute::AttributeID, value::value_type::ValueType};
+use encoding::value::value_type::ValueTypeCategory;
 use resource::constants::snapshot::BUFFER_VALUE_INLINE;
 use storage::snapshot::{ReadableSnapshot, WritableSnapshot};
 
 use crate::{
+    ConceptStatus,
     error::{ConceptReadError, ConceptWriteError},
     thing::thing_manager::ThingManager,
-    ConceptStatus,
 };
 
 pub mod attribute;
@@ -22,6 +23,7 @@ pub mod relation;
 pub mod statistics;
 pub mod thing_manager;
 pub mod value;
+mod value_struct;
 
 pub trait ThingAPI<'a> {
     fn set_modified<Snapshot: WritableSnapshot>(&self, snapshot: &mut Snapshot, thing_manager: &ThingManager<Snapshot>);
@@ -47,11 +49,11 @@ pub trait ThingAPI<'a> {
 }
 
 // TODO: where do these belong? They're encodings of values we store for keys
-pub(crate) fn decode_attribute_ids(value_type: ValueType, bytes: &[u8]) -> impl Iterator<Item = AttributeID> + '_ {
-    let chunk_size = AttributeID::value_type_encoding_length(value_type);
+pub(crate) fn decode_attribute_ids(value_type_category: ValueTypeCategory, bytes: &[u8]) -> impl Iterator<Item = AttributeID> + '_ {
+    let chunk_size = AttributeID::value_type_encoding_length(value_type_category);
     let chunks_iter = bytes.chunks_exact(chunk_size);
     debug_assert!(chunks_iter.remainder().is_empty());
-    chunks_iter.map(move |chunk| AttributeID::new(value_type, chunk))
+    chunks_iter.map(move |chunk| AttributeID::new(value_type_category, chunk))
 }
 
 pub(crate) fn encode_attribute_ids(attribute_ids: impl Iterator<Item = AttributeID>) -> ByteArray<BUFFER_VALUE_INLINE> {

concept/thing/thing_manager.rs

@@ -173,11 +173,12 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
         snapshot: &'this Snapshot,
         attribute_type: AttributeType<'_>,
     ) -> Result<AttributeIterator<'this, Snapshot>, ConceptReadError> {
-        let Some(value_type) = attribute_type.get_value_type(snapshot, self.type_manager.as_ref())? else {
+        let attribute_value_type = attribute_type.get_value_type(snapshot, self.type_manager.as_ref())?;
+        let Some(value_type) = attribute_value_type.as_ref() else {
             return Ok(AttributeIterator::new_empty());
         };
 
-        let attribute_value_type_prefix = AttributeVertex::value_type_to_prefix_type(value_type);
+        let attribute_value_type_prefix = AttributeVertex::value_type_category_to_prefix_type(value_type.category());
         let prefix =
             AttributeVertex::build_prefix_type(attribute_value_type_prefix, attribute_type.vertex().type_id_());
         let attribute_iterator =
@@ -191,12 +192,14 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
         Ok(AttributeIterator::new(attribute_iterator, has_reverse_iterator, snapshot, self.type_manager()))
     }
 
-    pub(crate) fn get_attribute_value(
+    pub(crate) fn get_attribute_value<'a>(
         &self,
         snapshot: &Snapshot,
-        attribute: &Attribute<'_>,
+        attribute: &'a Attribute<'a>,
     ) -> Result<Value<'static>, ConceptReadError> {
-        match attribute.value_type() {
+        let attribute_type = attribute.type_();
+        let value_type = attribute_type.get_value_type(snapshot, self.type_manager())?;
+        match value_type.as_ref().unwrap() {
             ValueType::Boolean => {
                 let attribute_id = attribute.vertex().attribute_id().unwrap_boolean();
                 Ok(Value::Boolean(BooleanBytes::new(attribute_id.bytes()).as_bool()))
@@ -228,6 +231,9 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
                         .unwrap())
                 }
             }
+            ValueType::Struct(definition_key) => {
+                todo!()
+            }
         }
     }
 
@@ -237,7 +243,13 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
         attribute_type: AttributeType<'static>,
         value: Value<'_>,
     ) -> Result<Option<Attribute<'static>>, ConceptReadError> {
-        let attribute = match value.value_type() {
+        let value_type = value.value_type();
+        let attribute_value_type = attribute_type.get_value_type(snapshot, self.type_manager())?;
+        if attribute_value_type.is_none() || attribute_value_type.as_ref().unwrap() != &value_type {
+            return Ok(None);
+        }
+
+        let attribute = match value_type {
             ValueType::Boolean | ValueType::Long | ValueType::Double | ValueType::DateTime => {
                 debug_assert!(AttributeID::is_inlineable(value.as_reference()));
                 match self.get_attribute_with_value_inline(snapshot, attribute_type, value) {
@@ -263,6 +275,9 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
                     }
                 }
             }
+            ValueType::Struct(definition_key) => {
+                todo!()
+            }
         };
 
         let is_independent = attribute.type_().is_independent(snapshot, self.type_manager())?;
@@ -280,8 +295,12 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
         value: Value<'_>,
     ) -> Result<Option<Attribute<'static>>, ConceptReadError> {
         debug_assert!(AttributeID::is_inlineable(value.as_reference()));
+        let attribute_value_type = attribute_type.get_value_type(snapshot, self.type_manager())?;
+        if attribute_value_type.is_none() || attribute_value_type.as_ref().unwrap() == &value.value_type() {
+            return Ok(None)
+        }
         let vertex = AttributeVertex::build(
-            value.value_type(),
+            attribute_value_type.as_ref().unwrap().category(),
             attribute_type.vertex().type_id_(),
             AttributeID::build_inline(value),
         );
@@ -297,10 +316,11 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
         attribute_type: AttributeType<'static>,
         value: Value<'_>,
     ) -> Result<bool, ConceptReadError> {
+        // TODO: check value type matches attribute value type
         let value_type = value.value_type();
         let vertex = if AttributeID::is_inlineable(value.as_reference()) {
             // don't need to do an extra lookup to get the attribute vertex - if it exists, it will have this ID
-            AttributeVertex::build(value_type, attribute_type.vertex().type_id_(), AttributeID::build_inline(value))
+            AttributeVertex::build(value_type.category(), attribute_type.vertex().type_id_(), AttributeID::build_inline(value))
         } else {
             // non-inline attributes require an extra lookup before checking for the has edge existence
             let attribute = self.get_attribute_with_value(snapshot, attribute_type, value)?;
@@ -335,13 +355,14 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
         owner: &impl ObjectAPI<'a>,
         attribute_type: AttributeType<'static>,
     ) -> Result<HasAttributeIterator, ConceptReadError> {
-        let value_type = match attribute_type.get_value_type(snapshot, self.type_manager())? {
+        let attribute_value_type = attribute_type.get_value_type(snapshot, self.type_manager())?;
+        let value_type = match attribute_value_type.as_ref() {
             None => {
                 todo!("Handle missing value type - for abstract attributes. Or assume this will never happen")
             }
             Some(value_type) => value_type,
         };
-        let prefix = ThingEdgeHas::prefix_from_object_to_type(owner.vertex(), value_type, attribute_type.into_vertex());
+        let prefix = ThingEdgeHas::prefix_from_object_to_type(owner.vertex(), value_type.category(), attribute_type.into_vertex());
         Ok(HasAttributeIterator::new(
             snapshot.iterate_range(KeyRange::new_within(prefix, ThingEdgeHas::FIXED_WIDTH_ENCODING)),
         ))
@@ -354,15 +375,16 @@ impl<Snapshot: ReadableSnapshot> ThingManager<Snapshot> {
         attribute_type: AttributeType<'static>,
     ) -> Result<Vec<Attribute<'static>>, ConceptReadError> {
         let key = HAS_ORDER_PROPERTY_FACTORY.build(owner.vertex(), attribute_type.vertex());
-        let value_type = match attribute_type.get_value_type(snapshot, self.type_manager())? {
+        let attribute_value_type = attribute_type.get_value_type(snapshot, self.type_manager())?;
+        let value_type = match attribute_value_type.as_ref() {
             None => {
                 todo!("Handle missing value type - for abstract attributes. Or assume this will never happen")
             }
             Some(value_type) => value_type,
         };
         let attributes = snapshot
             .get_mapped(key.as_storage_key().as_reference(), |bytes| {
-                decode_attribute_ids(value_type, bytes.bytes())
+                decode_attribute_ids(value_type.category(), bytes.bytes())
                     .map(|id| Attribute::new(AttributeVertex::new(Bytes::copy(id.bytes()))))
                     .collect()
             })
@@ -662,7 +684,7 @@ impl<'txn, Snapshot: WritableSnapshot> ThingManager<Snapshot> {
         value: Value<'_>,
     ) -> Result<Attribute<'a>, ConceptWriteError> {
         let value_type = attribute_type.get_value_type(snapshot, self.type_manager.as_ref())?;
-        if Some(value.value_type()) == value_type {
+        if Some(&value.value_type()) == value_type.as_ref() {
             let vertex = match value {
                 Value::Boolean(bool) => {
                     let encoded_boolean = BooleanBytes::build(bool);
@@ -719,10 +741,13 @@ impl<'txn, Snapshot: WritableSnapshot> ThingManager<Snapshot> {
                         .create_attribute_string(attribute_type.vertex().type_id_(), encoded_string, snapshot)
                         .map_err(|err| ConceptWriteError::SnapshotIterate { source: err })?
                 }
+                Value::Struct(struct_) => {
+                    todo!()
+                }
             };
             Ok(Attribute::new(vertex))
         } else {
-            Err(ConceptWriteError::ValueTypeMismatch { expected: value_type, provided: value.value_type() })
+            Err(ConceptWriteError::ValueTypeMismatch { expected: value_type.as_ref().cloned(), provided: value.value_type() })
         }
     }
 

concept/thing/value.rs

@@ -7,12 +7,14 @@
 use std::borrow::Cow;
 
 use chrono::NaiveDateTime;
+
 use encoding::value::{
     boolean_bytes::BooleanBytes, date_time_bytes::DateTimeBytes, double_bytes::DoubleBytes, long_bytes::LongBytes,
     string_bytes::StringBytes, value_type::ValueType, ValueEncodable,
 };
+use encoding::value::struct_bytes::StructBytes;
 
-// TODO: how do we handle user-created compound structs?
+use crate::thing::value_struct::StructValue;
 
 #[derive(Debug, Clone, PartialEq)]
 pub enum Value<'a> {
@@ -21,6 +23,7 @@ pub enum Value<'a> {
     Double(f64),
     DateTime(NaiveDateTime),
     String(Cow<'a, str>),
+    Struct(Cow<'a, StructValue<'static>>),
 }
 
 impl<'a> Value<'a> {
@@ -31,6 +34,7 @@ impl<'a> Value<'a> {
             Value::Double(double) => Value::Double(*double),
             Value::DateTime(date_time) => Value::DateTime(*date_time),
             Value::String(string) => Value::String(Cow::Borrowed(string.as_ref())),
+            Value::Struct(struct_) => Value::Struct(Cow::Borrowed(struct_.as_ref()))
         }
     }
 
@@ -68,6 +72,13 @@ impl<'a> Value<'a> {
             _ => panic!("Cannot unwrap String if not a string value."),
         }
     }
+
+    pub fn unwrap_struct(self) -> Cow<'a, StructValue<'static>> {
+        match self {
+            Value::Struct(struct_) => struct_,
+            _ => panic!("Cannot unwrap Struct if not a struct value.")
+        }
+    }
 }
 
 impl<'a> ValueEncodable for Value<'a> {
@@ -78,41 +89,51 @@ impl<'a> ValueEncodable for Value<'a> {
             Value::Double(_) => ValueType::Double,
             Value::DateTime(_) => ValueType::DateTime,
             Value::String(_) => ValueType::String,
+            Value::Struct(struct_value) => ValueType::Struct(struct_value.definition_key().into_owned())
         }
     }
 
     fn encode_boolean(&self) -> BooleanBytes {
         match self {
             Self::Boolean(boolean) => BooleanBytes::build(*boolean),
-            _ => panic!("Cannot encoded non-boolean as BooleanBytes"),
+            _ => panic!("Cannot encode non-boolean as BooleanBytes"),
         }
     }
 
     fn encode_long(&self) -> LongBytes {
         match self {
             Self::Long(long) => LongBytes::build(*long),
-            _ => panic!("Cannot encoded non-long as LongBytes"),
+            _ => panic!("Cannot encode non-long as LongBytes"),
         }
     }
 
     fn encode_double(&self) -> DoubleBytes {
         match self {
             Self::Double(double) => DoubleBytes::build(*double),
-            _ => panic!("Cannot encoded non-double as DoubleBytes"),
+            _ => panic!("Cannot encode non-double as DoubleBytes"),
         }
     }
 
     fn encode_date_time(&self) -> DateTimeBytes {
         match self {
             Self::DateTime(date_time) => DateTimeBytes::build(*date_time),
-            _ => panic!("Cannot encoded non-datetime as DateTimeBytes"),
+            _ => panic!("Cannot encode non-datetime as DateTimeBytes"),
         }
     }
 
     fn encode_string<const INLINE_LENGTH: usize>(&self) -> StringBytes<'_, INLINE_LENGTH> {
         match self {
             Value::String(str) => StringBytes::build_ref(str),
-            _ => panic!("Cannot encoded non-String as StringBytes"),
+            _ => panic!("Cannot encode non-String as StringBytes"),
+        }
+    }
+
+    fn encode_struct<const INLINE_LENGTH: usize>(&self) -> StructBytes<'static, INLINE_LENGTH> {
+        match self {
+            Value::Struct(struct_) => {
+                todo!()
+            },
+            _ => panic!("Cannot encode non-Struct as StructBytes"),
         }
     }
 }

concept/thing/value_struct.rs

@@ -0,0 +1,40 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+
+use std::collections::HashMap;
+use serde::{Deserialize, Serialize};
+
+use encoding::graph::definition::definition_key::DefinitionKey;
+use encoding::graph::type_::vertex::TypeVertex;
+use resource::constants::encoding::StructFieldIDUInt;
+
+use crate::thing::value::Value;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct StructValue<'a> {
+    definition: DefinitionKey<'static>,
+
+    // a map allows empty fields to not be recorded at all
+    fields: HashMap<StructFieldIDUInt, Value<'a>>
+}
+
+impl<'a> StructValue<'a> {
+
+    pub fn definition_key(&self) -> DefinitionKey<'_> {
+        self.definition.as_reference()
+    }
+}
+
+
+// TODO: implement serialise/deserialise for the StructValue
+//       since JSON serialisation seems to be able to handle recursive nesting, it should be able to handle that

concept/type_/attribute_type.rs

@@ -102,11 +102,11 @@ impl<'a> AttributeType<'a> {
         type_manager.storage_set_value_type(snapshot, self.clone().into_owned(), value_type)
     }
 
-    pub fn get_value_type<Snapshot: ReadableSnapshot>(
+    pub fn get_value_type<'m, Snapshot: ReadableSnapshot>(
         &self,
         snapshot: &Snapshot,
-        type_manager: &TypeManager<Snapshot>,
-    ) -> Result<Option<ValueType>, ConceptReadError> {
+        type_manager: &'m TypeManager<Snapshot>,
+    ) -> Result<MaybeOwns<'m, Option<ValueType>>, ConceptReadError> {
         type_manager.get_attribute_type_value_type(snapshot, self.clone().into_owned())
     }
 

concept/type_/type_cache/type_cache.rs

@@ -248,8 +248,8 @@ impl TypeCache {
         &T::get_cache(self, type_).owner_player_cache().plays_transitive
     }
 
-    pub(crate) fn get_attribute_type_value_type<'a>(&self, attribute_type: AttributeType<'a>) -> Option<ValueType> {
-        AttributeType::get_cache(&self, attribute_type).value_type
+    pub(crate) fn get_attribute_type_value_type<'a>(&self, attribute_type: AttributeType<'a>) -> &Option<ValueType> {
+        &AttributeType::get_cache(&self, attribute_type).value_type
     }
 
     pub(crate) fn get_owns_annotations<'c>(&'c self, owns: Owns<'c>) -> &'c HashSet<OwnsAnnotation> {