Custom Data: support implicit sharing for custom data layers

This integrates the new implicit-sharing system (from fbcddfc)
with `CustomData`. Now the potentially long arrays referenced by custom
data layers can be shared between different systems but most importantly
between different geometries. This makes e.g. copying a mesh much cheaper
because none of the attributes has to be copied. Only when an attribute
is modified does it have to be copied.

Also see the original design task: #95845.

This reduces memory and improves performance by avoiding unnecessary
data copies. For example, the used memory after loading a highly
subdivided mesh is reduced from 2.4GB to 1.79GB. This is about 25%
less which is the expected amount because in `main` there are 4 copies
of the data:
1. The original data which is allocated when the file is loaded.
2. The copy for the depsgraph allocated during depsgraph evaluation.
3. The copy for the undo system allocated when the first undo step is
  created right after loading the file.
4. GPU buffers allocated for drawing.

This patch only gets rid of copy number 2 for the depsgraph. In theory
the other copies can be removed as part of follow up PRs as well though.

-----

The patch has three main components:
* Slightly modified `CustomData` API to make it work better with implicit
  sharing:
  * `CD_REFERENCE` and `CD_DUPLICATE` have been removed because they are
    meaningless when implicit-sharing is used.
  * `CD_ASSIGN` has been removed as well because it's not an allocation
    type anyway. The functionality of using existing arrays as custom
    data layers has not been removed though.
  * This can still be done with `CustomData_add_layer_with_data` which
    also has a new argument that allows passing in information about
    whether the array is shared.
  * `CD_FLAG_NOFREE` has been removed because it's no longer necessary. It
    only existed because of `CD_REFERENCE`.
  * `CustomData_copy` and `CustomData_merge` have been split up into a
    functions that do copy the actual attribute values and those that do
    not. The latter functions now have the `_layout` suffix
    (e.g. `CustomData_copy_layout`).
* Changes in `customdata.cc` to make it actually use implicit-sharing.
* Changes in various other files to adapt to the changes in `BKE_customdata.h`.

Pull Request: https://projects.blender.org/blender/blender/pulls/106228

source/blender/blenkernel/BKE_attribute.hh

@@ -802,7 +802,6 @@ class CustomDataAttributes {
 
   std::optional<blender::GMutableSpan> get_for_write(const AttributeIDRef &attribute_id);
   bool create(const AttributeIDRef &attribute_id, eCustomDataType data_type);
-  bool create_by_move(const AttributeIDRef &attribute_id, eCustomDataType data_type, void *buffer);
   bool remove(const AttributeIDRef &attribute_id);
 
   bool foreach_attribute(const AttributeForeachCallback callback, eAttrDomain domain) const;

source/blender/blenkernel/BKE_customdata.h

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include "BLI_implicit_sharing.h"
 #include "BLI_sys_types.h"
 #include "BLI_utildefines.h"
 #ifdef __cplusplus
@@ -74,14 +75,8 @@ extern const CustomData_MeshMasks CD_MASK_EVERYTHING;
 
 /** Add/copy/merge allocation types. */
 typedef enum eCDAllocType {
-  /** Use the data pointer. */
-  CD_ASSIGN = 0,
   /** Allocate and set to default, which is usually just zeroed memory. */
   CD_SET_DEFAULT = 2,
-  /** Use data pointers, set layer flag NOFREE. */
-  CD_REFERENCE = 3,
-  /** Do a full copy of all layers, only allowed if source has same number of elements. */
-  CD_DUPLICATE = 4,
   /**
    * Default construct new layer values. Does nothing for trivial types. This should be used
    * if all layer values will be set by the caller after creating the layer.
@@ -158,28 +153,43 @@ void CustomData_data_multiply(eCustomDataType type, void *data, float fac);
 void CustomData_data_add(eCustomDataType type, void *data1, const void *data2);
 
 /**
- * Initializes a CustomData object with the same layer setup as source.
- * mask is a bit-field where `(mask & (1 << (layer type)))` indicates
- * if a layer should be copied or not. alloctype must be one of the above.
+ * Initializes a CustomData object with the same layer setup as source. `mask` is a bit-field where
+ * `(mask & (1 << (layer type)))` indicates if a layer should be copied or not. Data layers using
+ * implicit-sharing will not actually be copied but will be shared between source and destination.
  */
 void CustomData_copy(const struct CustomData *source,
                      struct CustomData *dest,
                      eCustomDataMask mask,
-                     eCDAllocType alloctype,
                      int totelem);
+/**
+ * Initializes a CustomData object with the same layers as source. The data is not copied from the
+ * source. Instead, the new layers are initialized using the given `alloctype`.
+ */
+void CustomData_copy_layout(const struct CustomData *source,
+                            struct CustomData *dest,
+                            eCustomDataMask mask,
+                            eCDAllocType alloctype,
+                            int totelem);
 
 /* BMESH_TODO, not really a public function but readfile.c needs it */
 void CustomData_update_typemap(struct CustomData *data);
 
 /**
- * Same as the above, except that this will preserve existing layers, and only
- * add the layers that were not there yet.
+ * Copies all layers from source to destination that don't exist there yet.
  */
 bool CustomData_merge(const struct CustomData *source,
                       struct CustomData *dest,
                       eCustomDataMask mask,
-                      eCDAllocType alloctype,
                       int totelem);
+/**
+ * Copies all layers from source to destination that don't exist there yet. The layer data is not
+ * copied. Instead the newly created layers are initialized using the given `alloctype`.
+ */
+bool CustomData_merge_layout(const struct CustomData *source,
+                             struct CustomData *dest,
+                             eCustomDataMask mask,
+                             eCDAllocType alloctype,
+                             int totelem);
 
 /**
  * Reallocate custom data to a new element count. If the new size is larger, the new values use
@@ -189,16 +199,16 @@ bool CustomData_merge(const struct CustomData *source,
 void CustomData_realloc(struct CustomData *data, int old_size, int new_size);
 
 /**
- * BMesh version of CustomData_merge; merges the layouts of source and `dest`,
+ * BMesh version of CustomData_merge_layout; merges the layouts of source and `dest`,
  * then goes through the mesh and makes sure all the custom-data blocks are
  * consistent with the new layout.
  */
-bool CustomData_bmesh_merge(const struct CustomData *source,
-                            struct CustomData *dest,
-                            eCustomDataMask mask,
-                            eCDAllocType alloctype,
-                            struct BMesh *bm,
-                            char htype);
+bool CustomData_bmesh_merge_layout(const struct CustomData *source,
+                                   struct CustomData *dest,
+                                   eCustomDataMask mask,
+                                   eCDAllocType alloctype,
+                                   struct BMesh *bm,
+                                   char htype);
 
 /**
  * Remove layers that aren't stored in BMesh or are stored as flags on BMesh.
@@ -229,18 +239,25 @@ void CustomData_free_typemask(struct CustomData *data, int totelem, eCustomDataM
 void CustomData_free_temporary(struct CustomData *data, int totelem);
 
 /**
- * Adds a data layer of the given type to the #CustomData object, optionally
- * backed by an external data array. the different allocation types are
- * defined above. returns the data of the layer.
+ * Adds a layer of the given type to the #CustomData object. The new layer is initialized based on
+ * the given alloctype.
+ * \return The layer data.
  */
 void *CustomData_add_layer(struct CustomData *data,
                            eCustomDataType type,
                            eCDAllocType alloctype,
                            int totelem);
+
+/**
+ * Adds a layer of the given type to the #CustomData object. The new layer takes ownership of the
+ * passed in `layer_data`. If a #ImplicitSharingInfoHandle is passed in, its user count is
+ * increased.
+ */
 const void *CustomData_add_layer_with_data(struct CustomData *data,
                                            eCustomDataType type,
                                            void *layer_data,
-                                           int totelem);
+                                           int totelem,
+                                           const ImplicitSharingInfoHandle *sharing_info);
 
 /**
  * Same as above but accepts a name.
@@ -250,17 +267,26 @@ void *CustomData_add_layer_named(struct CustomData *data,
                                  eCDAllocType alloctype,
                                  int totelem,
                                  const char *name);
+
 const void *CustomData_add_layer_named_with_data(struct CustomData *data,
                                                  eCustomDataType type,
                                                  void *layer_data,
                                                  int totelem,
-                                                 const char *name);
+                                                 const char *name,
+                                                 const ImplicitSharingInfoHandle *sharing_info);
+
 void *CustomData_add_layer_anonymous(struct CustomData *data,
                                      eCustomDataType type,
                                      eCDAllocType alloctype,
-                                     void *layer,
                                      int totelem,
                                      const AnonymousAttributeIDHandle *anonymous_id);
+const void *CustomData_add_layer_anonymous_with_data(
+    struct CustomData *data,
+    eCustomDataType type,
+    const AnonymousAttributeIDHandle *anonymous_id,
+    int totelem,
+    void *layer_data,
+    const ImplicitSharingInfoHandle *sharing_info);
 
 /**
  * Frees the active or first data layer with the give type.
@@ -296,11 +322,6 @@ int CustomData_number_of_layers(const struct CustomData *data, eCustomDataType t
 int CustomData_number_of_anonymous_layers(const struct CustomData *data, eCustomDataType type);
 int CustomData_number_of_layers_typemask(const struct CustomData *data, eCustomDataMask mask);
 
-/**
- * Duplicate all the layers with flag NOFREE, and remove the flag from duplicated layers.
- */
-void CustomData_duplicate_referenced_layers(CustomData *data, int totelem);
-
 /**
  * Set the #CD_FLAG_NOCOPY flag in custom data layers where the mask is
  * zero for the layer type, so only layer types specified by the mask will be copied

source/blender/blenkernel/intern/DerivedMesh.cc

@@ -207,11 +207,12 @@ void DM_from_template(DerivedMesh *dm,
                       int numPolys)
 {
   const CustomData_MeshMasks *mask = &CD_MASK_DERIVEDMESH;
-  CustomData_copy(&source->vertData, &dm->vertData, mask->vmask, CD_SET_DEFAULT, numVerts);
-  CustomData_copy(&source->edgeData, &dm->edgeData, mask->emask, CD_SET_DEFAULT, numEdges);
-  CustomData_copy(&source->faceData, &dm->faceData, mask->fmask, CD_SET_DEFAULT, numTessFaces);
-  CustomData_copy(&source->loopData, &dm->loopData, mask->lmask, CD_SET_DEFAULT, numLoops);
-  CustomData_copy(&source->polyData, &dm->polyData, mask->pmask, CD_SET_DEFAULT, numPolys);
+  CustomData_copy_layout(&source->vertData, &dm->vertData, mask->vmask, CD_SET_DEFAULT, numVerts);
+  CustomData_copy_layout(&source->edgeData, &dm->edgeData, mask->emask, CD_SET_DEFAULT, numEdges);
+  CustomData_copy_layout(
+      &source->faceData, &dm->faceData, mask->fmask, CD_SET_DEFAULT, numTessFaces);
+  CustomData_copy_layout(&source->loopData, &dm->loopData, mask->lmask, CD_SET_DEFAULT, numLoops);
+  CustomData_copy_layout(&source->polyData, &dm->polyData, mask->pmask, CD_SET_DEFAULT, numPolys);
   dm->poly_offsets = static_cast<int *>(MEM_dupallocN(source->poly_offsets));
 
   dm->type = type;

source/blender/blenkernel/intern/attribute_access.cc

@@ -201,13 +201,16 @@ static bool add_builtin_type_custom_data_layer_from_init(CustomData &custom_data
       return true;
     }
     case AttributeInit::Type::MoveArray: {
-      void *source_data = static_cast<const AttributeInitMoveArray &>(initializer).data;
-      const void *data = CustomData_add_layer_with_data(
-          &custom_data, data_type, source_data, domain_num);
-      if (data == nullptr) {
-        MEM_freeN(source_data);
+      void *src_data = static_cast<const AttributeInitMoveArray &>(initializer).data;
+      const void *stored_data = CustomData_add_layer_with_data(
+          &custom_data, data_type, src_data, domain_num, nullptr);
+      if (stored_data == nullptr) {
         return false;
       }
+      if (stored_data != src_data) {
+        MEM_freeN(src_data);
+        return true;
+      }
       return true;
     }
   }
@@ -230,25 +233,26 @@ static void *add_generic_custom_data_layer(CustomData &custom_data,
   }
   const AnonymousAttributeID &anonymous_id = attribute_id.anonymous_id();
   return CustomData_add_layer_anonymous(
-      &custom_data, data_type, alloctype, nullptr, domain_size, &anonymous_id);
+      &custom_data, data_type, alloctype, domain_size, &anonymous_id);
 }
 
 static const void *add_generic_custom_data_layer_with_existing_data(
     CustomData &custom_data,
     const eCustomDataType data_type,
-    void *layer_data,
+    const AttributeIDRef &attribute_id,
     const int domain_size,
-    const AttributeIDRef &attribute_id)
+    void *layer_data,
+    const ImplicitSharingInfo *sharing_info)
 {
-  if (!attribute_id.is_anonymous()) {
-    char attribute_name_c[MAX_CUSTOMDATA_LAYER_NAME];
-    attribute_id.name().copy(attribute_name_c);
-    return CustomData_add_layer_named_with_data(
-        &custom_data, data_type, layer_data, domain_size, attribute_name_c);
+  if (attribute_id.is_anonymous()) {
+    const AnonymousAttributeID &anonymous_id = attribute_id.anonymous_id();
+    return CustomData_add_layer_anonymous_with_data(
+        &custom_data, data_type, &anonymous_id, domain_size, layer_data, sharing_info);
   }
-  const AnonymousAttributeID &anonymous_id = attribute_id.anonymous_id();
-  return CustomData_add_layer_anonymous(
-      &custom_data, data_type, CD_ASSIGN, layer_data, domain_size, &anonymous_id);
+  char attribute_name_c[MAX_CUSTOMDATA_LAYER_NAME];
+  attribute_id.name().copy(attribute_name_c);
+  return CustomData_add_layer_named_with_data(
+      &custom_data, data_type, layer_data, domain_size, attribute_name_c, sharing_info);
 }
 
 static bool add_custom_data_layer_from_attribute_init(const AttributeIDRef &attribute_id,
@@ -279,9 +283,9 @@ static bool add_custom_data_layer_from_attribute_init(const AttributeIDRef &attr
       break;
     }
     case AttributeInit::Type::MoveArray: {
-      void *source_data = static_cast<const AttributeInitMoveArray &>(initializer).data;
+      void *data = static_cast<const AttributeInitMoveArray &>(initializer).data;
       add_generic_custom_data_layer_with_existing_data(
-          custom_data, data_type, source_data, domain_num, attribute_id);
+          custom_data, data_type, attribute_id, domain_num, data, nullptr);
       break;
     }
   }
@@ -571,7 +575,7 @@ CustomDataAttributes::~CustomDataAttributes()
 CustomDataAttributes::CustomDataAttributes(const CustomDataAttributes &other)
 {
   size_ = other.size_;
-  CustomData_copy(&other.data, &data, CD_MASK_ALL, CD_DUPLICATE, size_);
+  CustomData_copy(&other.data, &data, CD_MASK_ALL, size_);
 }
 
 CustomDataAttributes::CustomDataAttributes(CustomDataAttributes &&other)
@@ -655,15 +659,6 @@ bool CustomDataAttributes::create(const AttributeIDRef &attribute_id,
   return result != nullptr;
 }
 
-bool CustomDataAttributes::create_by_move(const AttributeIDRef &attribute_id,
-                                          const eCustomDataType data_type,
-                                          void *buffer)
-{
-  const void *result = add_generic_custom_data_layer_with_existing_data(
-      data, data_type, buffer, size_, attribute_id);
-  return result != nullptr;
-}
-
 bool CustomDataAttributes::remove(const AttributeIDRef &attribute_id)
 {
   for (const int i : IndexRange(data.totlayer)) {

source/blender/blenkernel/intern/cdderivedmesh.cc

@@ -147,9 +147,7 @@ static CDDerivedMesh *cdDM_create(const char *desc)
   return cddm;
 }
 
-static DerivedMesh *cdDM_from_mesh_ex(Mesh *mesh,
-                                      eCDAllocType alloctype,
-                                      const CustomData_MeshMasks *mask)
+static DerivedMesh *cdDM_from_mesh_ex(Mesh *mesh, const CustomData_MeshMasks *mask)
 {
   CDDerivedMesh *cddm = cdDM_create(__func__);
   DerivedMesh *dm = &cddm->dm;
@@ -167,15 +165,14 @@ static DerivedMesh *cdDM_from_mesh_ex(Mesh *mesh,
           mesh->totloop,
           mesh->totpoly);
 
-  CustomData_merge(&mesh->vdata, &dm->vertData, cddata_masks.vmask, alloctype, mesh->totvert);
-  CustomData_merge(&mesh->edata, &dm->edgeData, cddata_masks.emask, alloctype, mesh->totedge);
+  CustomData_merge(&mesh->vdata, &dm->vertData, cddata_masks.vmask, mesh->totvert);
+  CustomData_merge(&mesh->edata, &dm->edgeData, cddata_masks.emask, mesh->totedge);
   CustomData_merge(&mesh->fdata,
                    &dm->faceData,
                    cddata_masks.fmask | CD_MASK_ORIGINDEX,
-                   alloctype,
                    0 /* `mesh->totface` */);
-  CustomData_merge(&mesh->ldata, &dm->loopData, cddata_masks.lmask, alloctype, mesh->totloop);
-  CustomData_merge(&mesh->pdata, &dm->polyData, cddata_masks.pmask, alloctype, mesh->totpoly);
+  CustomData_merge(&mesh->ldata, &dm->loopData, cddata_masks.lmask, mesh->totloop);
+  CustomData_merge(&mesh->pdata, &dm->polyData, cddata_masks.pmask, mesh->totpoly);
 
   cddm->vert_positions = static_cast<float(*)[3]>(CustomData_get_layer_named_for_write(
       &dm->vertData, CD_PROP_FLOAT3, "position", mesh->totvert));
@@ -203,5 +200,5 @@ static DerivedMesh *cdDM_from_mesh_ex(Mesh *mesh,
 
 DerivedMesh *CDDM_from_mesh(Mesh *mesh)
 {
-  return cdDM_from_mesh_ex(mesh, CD_REFERENCE, &CD_MASK_MESH);
+  return cdDM_from_mesh_ex(mesh, &CD_MASK_MESH);
 }

source/blender/blenkernel/intern/curves.cc

@@ -62,7 +62,7 @@ static void curves_init_data(ID *id)
   new (&curves->geometry) blender::bke::CurvesGeometry();
 }
 
-static void curves_copy_data(Main * /*bmain*/, ID *id_dst, const ID *id_src, const int flag)
+static void curves_copy_data(Main * /*bmain*/, ID *id_dst, const ID *id_src, const int /*flag*/)
 {
   using namespace blender;
 
@@ -80,9 +80,8 @@ static void curves_copy_data(Main * /*bmain*/, ID *id_dst, const ID *id_src, con
   dst.point_num = src.point_num;
   dst.curve_num = src.curve_num;
 
-  const eCDAllocType alloc_type = (flag & LIB_ID_COPY_CD_REFERENCE) ? CD_REFERENCE : CD_DUPLICATE;
-  CustomData_copy(&src.point_data, &dst.point_data, CD_MASK_ALL, alloc_type, dst.point_num);
-  CustomData_copy(&src.curve_data, &dst.curve_data, CD_MASK_ALL, alloc_type, dst.curve_num);
+  CustomData_copy(&src.point_data, &dst.point_data, CD_MASK_ALL, dst.point_num);
+  CustomData_copy(&src.curve_data, &dst.curve_data, CD_MASK_ALL, dst.curve_num);
 
   dst.curve_offsets = static_cast<int *>(MEM_dupallocN(src.curve_offsets));
 

source/blender/blenkernel/intern/curves_geometry.cc

@@ -80,8 +80,8 @@ static void copy_curves_geometry(CurvesGeometry &dst, const CurvesGeometry &src)
   CustomData_free(&dst.curve_data, dst.curve_num);
   dst.point_num = src.point_num;
   dst.curve_num = src.curve_num;
-  CustomData_copy(&src.point_data, &dst.point_data, CD_MASK_ALL, CD_DUPLICATE, dst.point_num);
-  CustomData_copy(&src.curve_data, &dst.curve_data, CD_MASK_ALL, CD_DUPLICATE, dst.curve_num);
+  CustomData_copy(&src.point_data, &dst.point_data, CD_MASK_ALL, dst.point_num);
+  CustomData_copy(&src.curve_data, &dst.curve_data, CD_MASK_ALL, dst.curve_num);
 
   MEM_SAFE_FREE(dst.curve_offsets);
   dst.curve_offsets = (int *)MEM_malloc_arrayN(dst.point_num + 1, sizeof(int), __func__);

source/blender/blenkernel/intern/curves_utils.cc

@@ -101,11 +101,8 @@ void fill_points(const OffsetIndices<int> points_by_curve,
 bke::CurvesGeometry copy_only_curve_domain(const bke::CurvesGeometry &src_curves)
 {
   bke::CurvesGeometry dst_curves(0, src_curves.curves_num());
-  CustomData_copy(&src_curves.curve_data,
-                  &dst_curves.curve_data,
-                  CD_MASK_ALL,
-                  CD_DUPLICATE,
-                  src_curves.curves_num());
+  CustomData_copy(
+      &src_curves.curve_data, &dst_curves.curve_data, CD_MASK_ALL, src_curves.curves_num());
   dst_curves.runtime->type_counts = src_curves.runtime->type_counts;
   return dst_curves;
 }