Remove redundent information and optimize dynamic allocations in Table

benches/benches/bevy_ecs/components/add_remove_very_big_table.rs

@@ -0,0 +1,87 @@
+#![allow(dead_code)]
+
+use bevy_ecs::prelude::*;
+use glam::*;
+
+#[derive(Component, Copy, Clone)]
+struct A(Mat4);
+#[derive(Component, Copy, Clone)]
+struct B(Mat4);
+#[derive(Component, Copy, Clone)]
+struct C(Mat4);
+#[derive(Component, Copy, Clone)]
+struct D(Mat4);
+#[derive(Component, Copy, Clone)]
+struct E(Mat4);
+#[derive(Component, Copy, Clone)]
+struct F(Mat4);
+#[derive(Component, Copy, Clone)]
+struct A2(Mat4);
+#[derive(Component, Copy, Clone)]
+struct B2(Mat4);
+#[derive(Component, Copy, Clone)]
+struct C2(Mat4);
+#[derive(Component, Copy, Clone)]
+struct D2(Mat4);
+#[derive(Component, Copy, Clone)]
+struct E2(Mat4);
+#[derive(Component, Copy, Clone)]
+struct F2(Mat4);
+#[derive(Component, Copy, Clone)]
+struct A3(Mat4);
+#[derive(Component, Copy, Clone)]
+struct B3(Mat4);
+#[derive(Component, Copy, Clone)]
+struct C3(Mat4);
+#[derive(Component, Copy, Clone)]
+struct D3(Mat4);
+#[derive(Component, Copy, Clone)]
+struct E3(Mat4);
+#[derive(Component, Copy, Clone)]
+struct F3(Mat4);
+
+pub struct Benchmark(World, Vec<Entity>);
+
+impl Benchmark {
+    pub fn new() -> Self {
+        let mut world = World::default();
+        let mut entities = Vec::with_capacity(10_000);
+        for _ in 0..10_000 {
+            entities.push(
+                world
+                    .spawn((
+                        A(Mat4::from_scale(Vec3::ONE)),
+                        B(Mat4::from_scale(Vec3::ONE)),
+                        C(Mat4::from_scale(Vec3::ONE)),
+                        D(Mat4::from_scale(Vec3::ONE)),
+                        E(Mat4::from_scale(Vec3::ONE)),
+                        A2(Mat4::from_scale(Vec3::ONE)),
+                        B2(Mat4::from_scale(Vec3::ONE)),
+                        C2(Mat4::from_scale(Vec3::ONE)),
+                        D2(Mat4::from_scale(Vec3::ONE)),
+                        E2(Mat4::from_scale(Vec3::ONE)),
+                        A3(Mat4::from_scale(Vec3::ONE)),
+                        B3(Mat4::from_scale(Vec3::ONE)),
+                        C3(Mat4::from_scale(Vec3::ONE)),
+                        D3(Mat4::from_scale(Vec3::ONE)),
+                        E3(Mat4::from_scale(Vec3::ONE)),
+                    ))
+                    .id(),
+            );
+        }
+
+        Self(world, entities)
+    }
+
+    pub fn run(&mut self) {
+        for entity in &self.1 {
+            self.0
+                .entity_mut(*entity)
+                .insert(F(Mat4::from_scale(Vec3::ONE)));
+        }
+
+        for entity in &self.1 {
+            self.0.entity_mut(*entity).remove::<F>();
+        }
+    }
+}

benches/benches/bevy_ecs/components/mod.rs

@@ -4,6 +4,7 @@ mod add_remove_big_sparse_set;
 mod add_remove_big_table;
 mod add_remove_sparse_set;
 mod add_remove_table;
+mod add_remove_very_big_table;
 mod archetype_updates;
 mod insert_simple;
 mod insert_simple_unbatched;
@@ -14,6 +15,7 @@ criterion_group!(
     components_benches,
     add_remove,
     add_remove_big,
+    add_remove_very_big,
     insert_simple,
     no_archetypes,
     added_archetypes,
@@ -49,6 +51,17 @@ fn add_remove_big(c: &mut Criterion) {
     group.finish();
 }
 
+fn add_remove_very_big(c: &mut Criterion) {
+    let mut group = c.benchmark_group("add_remove_very_big");
+    group.warm_up_time(std::time::Duration::from_millis(500));
+    group.measurement_time(std::time::Duration::from_secs(4));
+    group.bench_function("table", |b| {
+        let mut bench = add_remove_very_big_table::Benchmark::new();
+        b.iter(move || bench.run());
+    });
+    group.finish();
+}
+
 fn insert_simple(c: &mut Criterion) {
     let mut group = c.benchmark_group("insert_simple");
     group.warm_up_time(std::time::Duration::from_millis(500));

crates/bevy_ecs/src/bundle.rs

@@ -390,19 +390,21 @@ impl BundleInfo {
             let component_id = *self.component_ids.get_unchecked(bundle_component);
             match storage_type {
                 StorageType::Table => {
-                    let column =
-                        // SAFETY: If component_id is in self.component_ids, BundleInfo::new requires that
-                        // the target table contains the component.
-                        unsafe { table.get_column_mut(component_id).debug_checked_unwrap() };
                     // SAFETY: bundle_component is a valid index for this bundle
                     let status = unsafe { bundle_component_status.get_status(bundle_component) };
                     match status {
-                        ComponentStatus::Added => {
-                            column.initialize(table_row, component_ptr, change_tick);
-                        }
-                        ComponentStatus::Mutated => {
-                            column.replace(table_row, component_ptr, change_tick);
-                        }
+                        ComponentStatus::Added => table.initialize_component(
+                            table_row,
+                            component_id,
+                            component_ptr,
+                            change_tick,
+                        ),
+                        ComponentStatus::Mutated => table.replace_component(
+                            table_row,
+                            component_id,
+                            component_ptr,
+                            change_tick,
+                        ),
                     }
                 }
                 StorageType::SparseSet => {

crates/bevy_ecs/src/lib.rs

@@ -600,7 +600,6 @@ mod tests {
                 .collect::<Vec<_>>(),
             &[(e1, A(1), B(3)), (e2, A(2), B(4))]
         );
-
         assert_eq!(world.entity_mut(e1).take::<A>(), Some(A(1)));
         assert_eq!(
             world

crates/bevy_ecs/src/query/fetch.rs

@@ -948,9 +948,8 @@ unsafe impl<T: Component> WorldQuery for &T {
     ) {
         fetch.table_components = Some(
             table
-                .get_column(component_id)
+                .get_column_data_slice(component_id)
                 .debug_checked_unwrap()
-                .get_data_slice()
                 .into(),
         );
     }
@@ -1106,11 +1105,19 @@ unsafe impl<'__w, T: Component> WorldQuery for Ref<'__w, T> {
         &component_id: &ComponentId,
         table: &'w Table,
     ) {
-        let column = table.get_column(component_id).debug_checked_unwrap();
         fetch.table_data = Some((
-            column.get_data_slice().into(),
-            column.get_added_ticks_slice().into(),
-            column.get_changed_ticks_slice().into(),
+            table
+                .get_column_data_slice(component_id)
+                .debug_checked_unwrap()
+                .into(),
+            table
+                .get_column_added_ticks(component_id)
+                .debug_checked_unwrap()
+                .into(),
+            table
+                .get_column_changed_ticks(component_id)
+                .debug_checked_unwrap()
+                .into(),
         ));
     }
 
@@ -1289,11 +1296,19 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T {
         &component_id: &ComponentId,
         table: &'w Table,
     ) {
-        let column = table.get_column(component_id).debug_checked_unwrap();
         fetch.table_data = Some((
-            column.get_data_slice().into(),
-            column.get_added_ticks_slice().into(),
-            column.get_changed_ticks_slice().into(),
+            table
+                .get_column_data_slice(component_id)
+                .debug_checked_unwrap()
+                .into(),
+            table
+                .get_column_added_ticks(component_id)
+                .debug_checked_unwrap()
+                .into(),
+            table
+                .get_column_changed_ticks(component_id)
+                .debug_checked_unwrap()
+                .into(),
         ));
     }
 

crates/bevy_ecs/src/query/filter.rs

@@ -3,7 +3,7 @@ use crate::{
     component::{Component, ComponentId, StorageType, Tick},
     entity::Entity,
     query::{DebugCheckedUnwrap, FilteredAccess, WorldQuery},
-    storage::{Column, ComponentSparseSet, Table, TableRow},
+    storage::{ComponentSparseSet, Table, TableRow},
     world::{unsafe_world_cell::UnsafeWorldCell, World},
 };
 use bevy_ptr::{ThinSlicePtr, UnsafeCellDeref};
@@ -70,7 +70,6 @@ use std::{cell::UnsafeCell, marker::PhantomData};
 /// [`matches_component_set`]: Self::matches_component_set
 /// [`Query`]: crate::system::Query
 /// [`State`]: Self::State
-
 pub trait QueryFilter: WorldQuery {
     /// Returns true if (and only if) this Filter relies strictly on archetypes to limit which
     /// components are accessed by the Query.
@@ -645,10 +644,9 @@ unsafe impl<T: Component> WorldQuery for Added<T> {
         &component_id: &ComponentId,
         table: &'w Table,
     ) {
-        fetch.table_ticks = Some(
-            Column::get_added_ticks_slice(table.get_column(component_id).debug_checked_unwrap())
-                .into(),
-        );
+        fetch.table_ticks = table
+            .get_column_added_ticks(component_id)
+            .map(|slice| slice.into());
     }
 
     #[inline(always)]
@@ -854,10 +852,9 @@ unsafe impl<T: Component> WorldQuery for Changed<T> {
         &component_id: &ComponentId,
         table: &'w Table,
     ) {
-        fetch.table_ticks = Some(
-            Column::get_changed_ticks_slice(table.get_column(component_id).debug_checked_unwrap())
-                .into(),
-        );
+        fetch.table_ticks = table
+            .get_column_changed_ticks(component_id)
+            .map(|slice| slice.into());
     }
 
     #[inline(always)]

crates/bevy_ecs/src/query/mod.rs

@@ -53,6 +53,40 @@ impl<T> DebugCheckedUnwrap for Option<T> {
     }
 }
 
+// These two impls are explicitly split to ensure that the unreachable! macro
+// does not cause inlining to fail when compiling in release mode.
+#[cfg(debug_assertions)]
+impl<T, U> DebugCheckedUnwrap for Result<T, U> {
+    type Item = T;
+
+    #[inline(always)]
+    #[track_caller]
+    unsafe fn debug_checked_unwrap(self) -> Self::Item {
+        if let Ok(inner) = self {
+            inner
+        } else {
+            unreachable!()
+        }
+    }
+}
+
+// These two impls are explicitly split to ensure that the unreachable! macro
+// does not cause inlining to fail when compiling in release mode.
+#[cfg(not(debug_assertions))]
+impl<T, U> DebugCheckedUnwrap for Result<T, U> {
+    type Item = T;
+
+    #[inline(always)]
+    #[track_caller]
+    unsafe fn debug_checked_unwrap(self) -> Self::Item {
+        if let Ok(inner) = self {
+            inner
+        } else {
+            std::hint::unreachable_unchecked()
+        }
+    }
+}
+
 #[cfg(not(debug_assertions))]
 impl<T> DebugCheckedUnwrap for Option<T> {
     type Item = T;